diff --git a/docs/pages/product/configuration.mdx b/docs/pages/product/configuration.mdx
index 855b8b53db87d..fd14340f55ccb 100644
--- a/docs/pages/product/configuration.mdx
+++ b/docs/pages/product/configuration.mdx
@@ -1,8 +1,3 @@
----
-redirect_from:
- - /configuration/overview
----
-
# Overview
Cube is configured via [environment variables][link-env-vars] and
@@ -103,6 +98,9 @@ module.exports = {
Both ways are equivalent; when in doubt, use Python.
+You can read more about [Python][ref-python] and [JavaScript][ref-javascript] support
+in the dynamic data modeling section of the documentation.
+
### Cube Core
When using Docker, ensure that the configuration file and your [data model
@@ -112,7 +110,8 @@ Docker container.
### Cube Cloud
You can edit the configuration file by going into Development Mode
-and navigating to the Data Model page.
+and navigating to [Data Model][ref-data-model] or [Visual
+Model][ref-visual-model] pages.
## Runtimes and dependencies
@@ -178,4 +177,8 @@ mode does the following:
[link-docker-env-vars]: https://docs.docker.com/compose/environment-variables/set-environment-variables/
[ref-mls]: /product/auth/member-level-security
[link-current-python-version]: https://github.com/cube-js/cube/blob/master/packages/cubejs-docker/latest.Dockerfile#L13
-[link-current-nodejs-version]: https://github.com/cube-js/cube/blob/master/packages/cubejs-docker/latest.Dockerfile#L1
\ No newline at end of file
+[link-current-nodejs-version]: https://github.com/cube-js/cube/blob/master/packages/cubejs-docker/latest.Dockerfile#L1
+[ref-data-model]: /product/workspace/data-model
+[ref-visual-model]: /product/workspace/visual-model
+[ref-python]: /product/data-modeling/dynamic/jinja#python
+[ref-javascript]: /product/data-modeling/dynamic/javascript
\ No newline at end of file
diff --git a/docs/pages/product/data-modeling/dynamic/jinja.mdx b/docs/pages/product/data-modeling/dynamic/jinja.mdx
index 7e65ff5151f75..56fec362d4e58 100644
--- a/docs/pages/product/data-modeling/dynamic/jinja.mdx
+++ b/docs/pages/product/data-modeling/dynamic/jinja.mdx
@@ -155,13 +155,20 @@ class SafeString(str):
## Python
-You can declare and invoke Python functions from within a Jinja template. This
-allows the reuse of existing code to generate data models. Cube uses Python 3.9 to execute Python code.
-It also installs packages listed in the `requirements.txt` with pip on the startup.
+### Template context
-These helper functions must be located in `model/globals.py` file or explicitly loaded from the YAML files.
-In the following example, we declare a function called `load_data()` which will load data from a remote
-API endpoint. We will then use the function to generate a data model in Cube.
+You can use Python to declare functions that can be invoked and variables that can be
+referenced from within a Jinja template. These functions and variables must be defined
+in `model/globals.py` file and registered in the `TemplateContext` instance.
+
+
+
+See the [`TemplateContext` reference][ref-cube-template-context] for more details.
+
+
+
+In the following example, we declare a function called `load_data` that supposedly loads
+data from a remote API endpoint. We will then use the function to generate a data model:
```python
from cube import TemplateContext
@@ -237,12 +244,46 @@ cubes:
{%- endfor %}
```
-
+### Imports
-If you'd like to split your Python code into several files, see
-[this issue](https://github.com/cube-js/cube/issues/8443#issuecomment-2219804266).
+In the `model/globals.py` file (or the `cube.py` configuration file), you can
+import modules from the current directory. In the following example, we import a function
+from the `utils` module and use it to populate a variable in the template context:
-
+```python filename="model/utils.py"
+def answer_to_main_question() -> str:
+ return "42"
+```
+
+```python filename="model/globals.py"
+from cube import TemplateContext
+from utils import answer_to_main_question
+
+template = TemplateContext()
+
+answer = answer_to_main_question()
+template.add_variable('answer', answer)
+```
+### Dependencies
+
+If you need to use dependencies in your dynamic data model (or your `cube.py`
+configuration file), you can list them in the `requirements.txt` file in the root
+directory of your Cube deployment. They will be automatically installed with `pip` on
+the startup.
+
+
+
+[`cube` package][ref-cube-package] is available out of the box, it doesn't need to be
+listed in `requirements.txt`.
+
+
+
+If you use dbt for data transformation, you might find the [`cube_dbt`
+package][ref-cube-dbt-package] useful. It provides a set of utilities that simplify
+defining the data model in YAML [based on dbt models][ref-cube-with-dbt].
+
+If you need to use dependencies with native extensions, build a [custom Docker
+image][ref-docker-image-extension].
[jinja]: https://jinja.palletsprojects.com/
@@ -253,4 +294,9 @@ If you'd like to split your Python code into several files, see
[jinja-docs-autoescaping]: https://jinja.palletsprojects.com/en/3.1.x/api/#autoescaping
[jinja-docs-filters-safe]: https://jinja.palletsprojects.com/en/3.1.x/templates/#jinja-filters.safe
[ref-cube-dbt]: /reference/python/cube_dbt
-[ref-visual-model]: /product/workspace/visual-model
\ No newline at end of file
+[ref-visual-model]: /product/workspace/visual-model
+[ref-docker-image-extension]: /product/deployment/core#extend-the-docker-image
+[ref-cube-package]: /reference/python/cube
+[ref-cube-template-context]: /reference/python/cube#templatecontext-class
+[ref-cube-dbt-package]: /reference/python/cube_dbt
+[ref-cube-with-dbt]: /guides/dbt
\ No newline at end of file
diff --git a/packages/cubejs-backend-native/src/python/entry.rs b/packages/cubejs-backend-native/src/python/entry.rs
index ce2e90634f4e6..e3ccae1a211fd 100644
--- a/packages/cubejs-backend-native/src/python/entry.rs
+++ b/packages/cubejs-backend-native/src/python/entry.rs
@@ -6,6 +6,7 @@ use crate::python::runtime::py_runtime_init;
use neon::prelude::*;
use pyo3::prelude::*;
use pyo3::types::{PyDict, PyFunction, PyList, PyString, PyTuple};
+use std::path::Path;
fn python_load_config(mut cx: FunctionContext) -> JsResult {
let file_content_arg = cx.argument::(0)?.value(&mut cx);
@@ -20,6 +21,15 @@ fn python_load_config(mut cx: FunctionContext) -> JsResult {
py_runtime_init(&mut cx, channel.clone())?;
let conf_res = Python::with_gil(|py| -> PyResult {
+ let sys_path = py.import("sys")?.getattr("path")?.downcast::()?;
+
+ let config_dir = Path::new(&options_file_name)
+ .parent()
+ .unwrap_or_else(|| Path::new("."));
+ let config_dir_str = config_dir.to_str().unwrap_or(".");
+
+ sys_path.insert(0, PyString::new(py, config_dir_str))?;
+
let cube_code = include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/python/cube/src/__init__.py"
@@ -61,6 +71,15 @@ fn python_load_model(mut cx: FunctionContext) -> JsResult {
py_runtime_init(&mut cx, channel.clone())?;
let conf_res = Python::with_gil(|py| -> PyResult {
+ let sys_path = py.import("sys")?.getattr("path")?.downcast::()?;
+
+ let config_dir = Path::new(&model_file_name)
+ .parent()
+ .unwrap_or_else(|| Path::new("."));
+ let config_dir_str = config_dir.to_str().unwrap_or(".");
+
+ sys_path.insert(0, PyString::new(py, config_dir_str))?;
+
let cube_code = include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/python/cube/src/__init__.py"
diff --git a/packages/cubejs-backend-native/test/config.py b/packages/cubejs-backend-native/test/config.py
index db6d9c64d98b6..4e1035532e089 100644
--- a/packages/cubejs-backend-native/test/config.py
+++ b/packages/cubejs-backend-native/test/config.py
@@ -1,4 +1,5 @@
from cube import config, file_repository
+from utils import test_function
config.schema_path = "models"
config.pg_sql_port = 5555
@@ -7,6 +8,7 @@
@config
def query_rewrite(query, ctx):
+ query = test_function(query)
print("[python] query_rewrite query=", query, " ctx=", ctx)
return query
diff --git a/packages/cubejs-backend-native/test/globals.py b/packages/cubejs-backend-native/test/globals.py
new file mode 100644
index 0000000000000..5ce5267e08d64
--- /dev/null
+++ b/packages/cubejs-backend-native/test/globals.py
@@ -0,0 +1,16 @@
+from cube import TemplateContext
+import os
+from utils import answer_to_main_question
+from subdir_for_test.meta import main_question
+
+
+template = TemplateContext()
+
+value_or_none = os.getenv('MY_ENV_VAR')
+template.add_variable('value_or_none', value_or_none)
+
+value_or_default = os.getenv('MY_OTHER_ENV_VAR', 'my_default_value')
+template.add_variable('value_or_default', value_or_default)
+
+template.add_variable('main_question', main_question())
+template.add_variable('answer_to_main_question', answer_to_main_question())
diff --git a/packages/cubejs-backend-native/test/globals_w_import_path.py b/packages/cubejs-backend-native/test/globals_w_import_path.py
new file mode 100644
index 0000000000000..e8eda386e252d
--- /dev/null
+++ b/packages/cubejs-backend-native/test/globals_w_import_path.py
@@ -0,0 +1,17 @@
+from cube import TemplateContext
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+
+from utils import answer_to_main_question
+
+template = TemplateContext()
+
+value_or_none = os.getenv('MY_ENV_VAR')
+template.add_variable('value_or_none', value_or_none)
+
+value_or_default = os.getenv('MY_OTHER_ENV_VAR', 'my_default_value')
+template.add_variable('value_or_default', value_or_default)
+
+template.add_variable('answer_to_main_question', answer_to_main_question())
diff --git a/packages/cubejs-backend-native/test/python.test.ts b/packages/cubejs-backend-native/test/python.test.ts
index 8059b1ecf715c..b13972a3f187c 100644
--- a/packages/cubejs-backend-native/test/python.test.ts
+++ b/packages/cubejs-backend-native/test/python.test.ts
@@ -9,16 +9,17 @@ const suite = native.isFallbackBuild() ? xdescribe : describe;
const darwinSuite = process.platform === 'darwin' && !native.isFallbackBuild() ? describe : xdescribe;
async function loadConfigurationFile(fileName: string) {
- const content = await fs.readFile(path.join(process.cwd(), 'test', fileName), 'utf8');
+ const fullFileName = path.join(process.cwd(), 'test', fileName);
+ const content = await fs.readFile(fullFileName, 'utf8');
console.log('content', {
content,
- fileName
+ fileName: fullFileName
});
const config = await native.pythonLoadConfig(
content,
{
- fileName
+ fileName: fullFileName
}
);
@@ -27,6 +28,26 @@ async function loadConfigurationFile(fileName: string) {
return config;
}
+const nativeInstance = new native.NativeInstance();
+
+suite('Python Models', () => {
+ test('models import', async () => {
+ const fullFileName = path.join(process.cwd(), 'test', 'globals.py');
+ const content = await fs.readFile(fullFileName, 'utf8');
+
+ // Just checking it won't fail
+ await nativeInstance.loadPythonContext(fullFileName, content);
+ });
+
+ test('models import with sys.path changed', async () => {
+ const fullFileName = path.join(process.cwd(), 'test', 'globals_w_import_path.py');
+ const content = await fs.readFile(fullFileName, 'utf8');
+
+ // Just checking it won't fail
+ await nativeInstance.loadPythonContext(fullFileName, content);
+ });
+});
+
suite('Python Config', () => {
let config: PyConfiguration;
diff --git a/packages/cubejs-backend-native/test/subdir_for_test/meta.py b/packages/cubejs-backend-native/test/subdir_for_test/meta.py
new file mode 100644
index 0000000000000..ce9a847d9fa0f
--- /dev/null
+++ b/packages/cubejs-backend-native/test/subdir_for_test/meta.py
@@ -0,0 +1,8 @@
+# Separate file module for testing python imports
+
+# Simple test function
+def test_meta_function(query: dict) -> dict:
+ return query
+
+def main_question() -> str:
+ return "Why?"
diff --git a/packages/cubejs-backend-native/test/utils.py b/packages/cubejs-backend-native/test/utils.py
new file mode 100644
index 0000000000000..e537899662df9
--- /dev/null
+++ b/packages/cubejs-backend-native/test/utils.py
@@ -0,0 +1,8 @@
+# Separate file module for testing python imports
+
+# Simple test function
+def test_function(query: dict) -> dict:
+ return query
+
+def answer_to_main_question() -> str:
+ return "42"