diff --git a/docs/pages/product/configuration.mdx b/docs/pages/product/configuration.mdx index 855b8b53db87d..fd14340f55ccb 100644 --- a/docs/pages/product/configuration.mdx +++ b/docs/pages/product/configuration.mdx @@ -1,8 +1,3 @@ ---- -redirect_from: - - /configuration/overview ---- - # Overview Cube is configured via [environment variables][link-env-vars] and @@ -103,6 +98,9 @@ module.exports = { Both ways are equivalent; when in doubt, use Python. +You can read more about [Python][ref-python] and [JavaScript][ref-javascript] support +in the dynamic data modeling section of the documentation. + ### Cube Core When using Docker, ensure that the configuration file and your [data model @@ -112,7 +110,8 @@ Docker container. ### Cube Cloud You can edit the configuration file by going into Development Mode -and navigating to the Data Model page. +and navigating to [Data Model][ref-data-model] or [Visual +Model][ref-visual-model] pages. ## Runtimes and dependencies @@ -178,4 +177,8 @@ mode does the following: [link-docker-env-vars]: https://docs.docker.com/compose/environment-variables/set-environment-variables/ [ref-mls]: /product/auth/member-level-security [link-current-python-version]: https://github.com/cube-js/cube/blob/master/packages/cubejs-docker/latest.Dockerfile#L13 -[link-current-nodejs-version]: https://github.com/cube-js/cube/blob/master/packages/cubejs-docker/latest.Dockerfile#L1 \ No newline at end of file +[link-current-nodejs-version]: https://github.com/cube-js/cube/blob/master/packages/cubejs-docker/latest.Dockerfile#L1 +[ref-data-model]: /product/workspace/data-model +[ref-visual-model]: /product/workspace/visual-model +[ref-python]: /product/data-modeling/dynamic/jinja#python +[ref-javascript]: /product/data-modeling/dynamic/javascript \ No newline at end of file diff --git a/docs/pages/product/data-modeling/dynamic/jinja.mdx b/docs/pages/product/data-modeling/dynamic/jinja.mdx index 7e65ff5151f75..56fec362d4e58 100644 --- a/docs/pages/product/data-modeling/dynamic/jinja.mdx +++ b/docs/pages/product/data-modeling/dynamic/jinja.mdx @@ -155,13 +155,20 @@ class SafeString(str): ## Python -You can declare and invoke Python functions from within a Jinja template. This -allows the reuse of existing code to generate data models. Cube uses Python 3.9 to execute Python code. -It also installs packages listed in the `requirements.txt` with pip on the startup. +### Template context -These helper functions must be located in `model/globals.py` file or explicitly loaded from the YAML files. -In the following example, we declare a function called `load_data()` which will load data from a remote -API endpoint. We will then use the function to generate a data model in Cube. +You can use Python to declare functions that can be invoked and variables that can be +referenced from within a Jinja template. These functions and variables must be defined +in `model/globals.py` file and registered in the `TemplateContext` instance. + + + +See the [`TemplateContext` reference][ref-cube-template-context] for more details. + + + +In the following example, we declare a function called `load_data` that supposedly loads +data from a remote API endpoint. We will then use the function to generate a data model: ```python from cube import TemplateContext @@ -237,12 +244,46 @@ cubes: {%- endfor %} ``` - +### Imports -If you'd like to split your Python code into several files, see -[this issue](https://github.com/cube-js/cube/issues/8443#issuecomment-2219804266). +In the `model/globals.py` file (or the `cube.py` configuration file), you can +import modules from the current directory. In the following example, we import a function +from the `utils` module and use it to populate a variable in the template context: - +```python filename="model/utils.py" +def answer_to_main_question() -> str: + return "42" +``` + +```python filename="model/globals.py" +from cube import TemplateContext +from utils import answer_to_main_question + +template = TemplateContext() + +answer = answer_to_main_question() +template.add_variable('answer', answer) +``` +### Dependencies + +If you need to use dependencies in your dynamic data model (or your `cube.py` +configuration file), you can list them in the `requirements.txt` file in the root +directory of your Cube deployment. They will be automatically installed with `pip` on +the startup. + + + +[`cube` package][ref-cube-package] is available out of the box, it doesn't need to be +listed in `requirements.txt`. + + + +If you use dbt for data transformation, you might find the [`cube_dbt` +package][ref-cube-dbt-package] useful. It provides a set of utilities that simplify +defining the data model in YAML [based on dbt models][ref-cube-with-dbt]. + +If you need to use dependencies with native extensions, build a [custom Docker +image][ref-docker-image-extension]. [jinja]: https://jinja.palletsprojects.com/ @@ -253,4 +294,9 @@ If you'd like to split your Python code into several files, see [jinja-docs-autoescaping]: https://jinja.palletsprojects.com/en/3.1.x/api/#autoescaping [jinja-docs-filters-safe]: https://jinja.palletsprojects.com/en/3.1.x/templates/#jinja-filters.safe [ref-cube-dbt]: /reference/python/cube_dbt -[ref-visual-model]: /product/workspace/visual-model \ No newline at end of file +[ref-visual-model]: /product/workspace/visual-model +[ref-docker-image-extension]: /product/deployment/core#extend-the-docker-image +[ref-cube-package]: /reference/python/cube +[ref-cube-template-context]: /reference/python/cube#templatecontext-class +[ref-cube-dbt-package]: /reference/python/cube_dbt +[ref-cube-with-dbt]: /guides/dbt \ No newline at end of file diff --git a/packages/cubejs-backend-native/src/python/entry.rs b/packages/cubejs-backend-native/src/python/entry.rs index ce2e90634f4e6..e3ccae1a211fd 100644 --- a/packages/cubejs-backend-native/src/python/entry.rs +++ b/packages/cubejs-backend-native/src/python/entry.rs @@ -6,6 +6,7 @@ use crate::python::runtime::py_runtime_init; use neon::prelude::*; use pyo3::prelude::*; use pyo3::types::{PyDict, PyFunction, PyList, PyString, PyTuple}; +use std::path::Path; fn python_load_config(mut cx: FunctionContext) -> JsResult { let file_content_arg = cx.argument::(0)?.value(&mut cx); @@ -20,6 +21,15 @@ fn python_load_config(mut cx: FunctionContext) -> JsResult { py_runtime_init(&mut cx, channel.clone())?; let conf_res = Python::with_gil(|py| -> PyResult { + let sys_path = py.import("sys")?.getattr("path")?.downcast::()?; + + let config_dir = Path::new(&options_file_name) + .parent() + .unwrap_or_else(|| Path::new(".")); + let config_dir_str = config_dir.to_str().unwrap_or("."); + + sys_path.insert(0, PyString::new(py, config_dir_str))?; + let cube_code = include_str!(concat!( env!("CARGO_MANIFEST_DIR"), "/python/cube/src/__init__.py" @@ -61,6 +71,15 @@ fn python_load_model(mut cx: FunctionContext) -> JsResult { py_runtime_init(&mut cx, channel.clone())?; let conf_res = Python::with_gil(|py| -> PyResult { + let sys_path = py.import("sys")?.getattr("path")?.downcast::()?; + + let config_dir = Path::new(&model_file_name) + .parent() + .unwrap_or_else(|| Path::new(".")); + let config_dir_str = config_dir.to_str().unwrap_or("."); + + sys_path.insert(0, PyString::new(py, config_dir_str))?; + let cube_code = include_str!(concat!( env!("CARGO_MANIFEST_DIR"), "/python/cube/src/__init__.py" diff --git a/packages/cubejs-backend-native/test/config.py b/packages/cubejs-backend-native/test/config.py index db6d9c64d98b6..4e1035532e089 100644 --- a/packages/cubejs-backend-native/test/config.py +++ b/packages/cubejs-backend-native/test/config.py @@ -1,4 +1,5 @@ from cube import config, file_repository +from utils import test_function config.schema_path = "models" config.pg_sql_port = 5555 @@ -7,6 +8,7 @@ @config def query_rewrite(query, ctx): + query = test_function(query) print("[python] query_rewrite query=", query, " ctx=", ctx) return query diff --git a/packages/cubejs-backend-native/test/globals.py b/packages/cubejs-backend-native/test/globals.py new file mode 100644 index 0000000000000..5ce5267e08d64 --- /dev/null +++ b/packages/cubejs-backend-native/test/globals.py @@ -0,0 +1,16 @@ +from cube import TemplateContext +import os +from utils import answer_to_main_question +from subdir_for_test.meta import main_question + + +template = TemplateContext() + +value_or_none = os.getenv('MY_ENV_VAR') +template.add_variable('value_or_none', value_or_none) + +value_or_default = os.getenv('MY_OTHER_ENV_VAR', 'my_default_value') +template.add_variable('value_or_default', value_or_default) + +template.add_variable('main_question', main_question()) +template.add_variable('answer_to_main_question', answer_to_main_question()) diff --git a/packages/cubejs-backend-native/test/globals_w_import_path.py b/packages/cubejs-backend-native/test/globals_w_import_path.py new file mode 100644 index 0000000000000..e8eda386e252d --- /dev/null +++ b/packages/cubejs-backend-native/test/globals_w_import_path.py @@ -0,0 +1,17 @@ +from cube import TemplateContext +import sys +import os + +sys.path.append(os.path.dirname(os.path.abspath(__file__))) + +from utils import answer_to_main_question + +template = TemplateContext() + +value_or_none = os.getenv('MY_ENV_VAR') +template.add_variable('value_or_none', value_or_none) + +value_or_default = os.getenv('MY_OTHER_ENV_VAR', 'my_default_value') +template.add_variable('value_or_default', value_or_default) + +template.add_variable('answer_to_main_question', answer_to_main_question()) diff --git a/packages/cubejs-backend-native/test/python.test.ts b/packages/cubejs-backend-native/test/python.test.ts index 8059b1ecf715c..b13972a3f187c 100644 --- a/packages/cubejs-backend-native/test/python.test.ts +++ b/packages/cubejs-backend-native/test/python.test.ts @@ -9,16 +9,17 @@ const suite = native.isFallbackBuild() ? xdescribe : describe; const darwinSuite = process.platform === 'darwin' && !native.isFallbackBuild() ? describe : xdescribe; async function loadConfigurationFile(fileName: string) { - const content = await fs.readFile(path.join(process.cwd(), 'test', fileName), 'utf8'); + const fullFileName = path.join(process.cwd(), 'test', fileName); + const content = await fs.readFile(fullFileName, 'utf8'); console.log('content', { content, - fileName + fileName: fullFileName }); const config = await native.pythonLoadConfig( content, { - fileName + fileName: fullFileName } ); @@ -27,6 +28,26 @@ async function loadConfigurationFile(fileName: string) { return config; } +const nativeInstance = new native.NativeInstance(); + +suite('Python Models', () => { + test('models import', async () => { + const fullFileName = path.join(process.cwd(), 'test', 'globals.py'); + const content = await fs.readFile(fullFileName, 'utf8'); + + // Just checking it won't fail + await nativeInstance.loadPythonContext(fullFileName, content); + }); + + test('models import with sys.path changed', async () => { + const fullFileName = path.join(process.cwd(), 'test', 'globals_w_import_path.py'); + const content = await fs.readFile(fullFileName, 'utf8'); + + // Just checking it won't fail + await nativeInstance.loadPythonContext(fullFileName, content); + }); +}); + suite('Python Config', () => { let config: PyConfiguration; diff --git a/packages/cubejs-backend-native/test/subdir_for_test/meta.py b/packages/cubejs-backend-native/test/subdir_for_test/meta.py new file mode 100644 index 0000000000000..ce9a847d9fa0f --- /dev/null +++ b/packages/cubejs-backend-native/test/subdir_for_test/meta.py @@ -0,0 +1,8 @@ +# Separate file module for testing python imports + +# Simple test function +def test_meta_function(query: dict) -> dict: + return query + +def main_question() -> str: + return "Why?" diff --git a/packages/cubejs-backend-native/test/utils.py b/packages/cubejs-backend-native/test/utils.py new file mode 100644 index 0000000000000..e537899662df9 --- /dev/null +++ b/packages/cubejs-backend-native/test/utils.py @@ -0,0 +1,8 @@ +# Separate file module for testing python imports + +# Simple test function +def test_function(query: dict) -> dict: + return query + +def answer_to_main_question() -> str: + return "42"