diff --git a/.github/actions/integration/athena.sh b/.github/actions/integration/athena.sh index 97b5eb5472263..f99d7b92b8723 100755 --- a/.github/actions/integration/athena.sh +++ b/.github/actions/integration/athena.sh @@ -5,6 +5,9 @@ set -eo pipefail export DEBUG=testcontainers echo "::group::Athena [cloud]" +export CUBEJS_AWS_KEY=$DRIVERS_TESTS_SNOWFLAKE_CUBEJS_AWS_KEY +export CUBEJS_AWS_SECRET=$DRIVERS_TESTS_SNOWFLAKE_CUBEJS_AWS_SECRET + export CUBEJS_AWS_REGION=us-east-1 export CUBEJS_AWS_S3_OUTPUT_LOCATION=s3://cubejs-opensource/testing/output export CUBEJS_DB_EXPORT_BUCKET=s3://cubejs-opensource/testing/export/ diff --git a/.github/actions/integration/bigquery.sh b/.github/actions/integration/bigquery.sh index 3da50e4862455..9cb19b57dcecd 100755 --- a/.github/actions/integration/bigquery.sh +++ b/.github/actions/integration/bigquery.sh @@ -5,6 +5,7 @@ set -eo pipefail export DEBUG=testcontainers echo "::group::BigQuery [cloud]" +export CUBEJS_DB_BQ_CREDENTIALS=$DRIVERS_TESTS_SNOWFLAKE_CUBEJS_DB_BQ_CREDENTIALS export CUBEJS_DB_BQ_PROJECT_ID=cube-open-source export CUBEJS_DB_EXPORT_BUCKET=cube-open-source-export-bucket yarn lerna run --concurrency 1 --stream --no-prefix integration:bigquery diff --git a/.github/actions/integration/snowflake.sh b/.github/actions/integration/snowflake.sh index 368030bfc09a3..7a30dc062a699 100755 --- a/.github/actions/integration/snowflake.sh +++ b/.github/actions/integration/snowflake.sh @@ -9,6 +9,8 @@ export CUBEJS_DB_NAME=DEMO_DB export CUBEJS_DB_SNOWFLAKE_ACCOUNT=lxb31104 export CUBEJS_DB_SNOWFLAKE_REGION=us-west-2 export CUBEJS_DB_SNOWFLAKE_WAREHOUSE=COMPUTE_WH +export CUBEJS_DB_USER=$DRIVERS_TESTS_SNOWFLAKE_CUBEJS_DB_USER +export CUBEJS_DB_PASS=$DRIVERS_TESTS_SNOWFLAKE_CUBEJS_DB_PASS yarn lerna run --concurrency 1 --stream --no-prefix smoke:snowflake diff --git a/.github/workflows/cloud.yml b/.github/workflows/cloud.yml deleted file mode 100644 index 389d92853a970..0000000000000 --- a/.github/workflows/cloud.yml +++ /dev/null @@ -1,130 +0,0 @@ -# CAUTION: This workflow exposes secrets. It is only supposed to be run on "merge into master" condition. -name: 'Cloud Tests' -on: - # push doesn't work for PRs, it's safe to use it - # only maintainers/core contributors are allowed to push directly to the main repository - push: - paths: - - '.github/actions/integration/athena.sh' - - '.github/actions/integration/bigquery.sh' - - '.github/actions/integration/snowflake.sh' - - '.github/workflows/cloud.yml' - - 'packages/cubejs-testing/**' - - 'packages/cubejs-testing-shared/**' - - 'packages/cubejs-backend-shared/**' - - 'packages/cubejs-base-driver/**' - - 'packages/cubejs-query-orchestrator/**' - - 'packages/cubejs-schema-compiler/**' - - 'packages/cubejs-backend-shared/**' - - 'packages/cubejs-server-core/**' - # drivers - - 'packages/cubejs-athena-driver/**' - - 'packages/cubejs-bigquery-driver/**' - - 'packages/cubejs-snowflake-driver/**' - - 'lerna.json' - - 'package.json' - - 'yarn.lock' - -env: - CUBEJS_TESSERACT_ORCHESTRATOR: true - -jobs: - latest-tag-sha: - runs-on: ubuntu-24.04 - outputs: - sha: ${{ steps.get-tag.outputs.sha }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - id: git-log - run: git log HEAD~30..HEAD - - id: get-tag-test - run: echo "$SHA $(git rev-list -n 1 "$(git tag --contains "$SHA")")" - env: - SHA: ${{ github.sha }} - - id: get-tag - run: echo "sha=$(git rev-list -n 1 "$(git tag --contains "$SHA")")" >> "$GITHUB_OUTPUT" - env: - SHA: ${{ github.sha }} - - id: get-tag-out - run: echo "$OUT" - env: - OUT: ${{ steps.get-tag.outputs.sha }} - - integration-cloud: - needs: latest-tag-sha - if: (needs['latest-tag-sha'].outputs.sha != github.sha) - runs-on: ubuntu-24.04 - timeout-minutes: 60 - - strategy: - matrix: - node-version: [ 22.x ] - db: [ 'athena', 'bigquery', 'snowflake' ] - target: [ "x86_64-unknown-linux-gnu" ] - fail-fast: false - - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Install Rust - uses: actions-rust-lang/setup-rust-toolchain@v1 - with: - toolchain: 1.84.1 - # override: true # this is by default on - rustflags: "" - components: rustfmt - target: ${{ matrix.target }} - - name: Install Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} - - name: Get yarn cache directory path - id: yarn-cache-dir-path - run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT" - shell: bash - - name: Restore yarn cache - uses: actions/cache@v4 - with: - path: ${{ steps.yarn-cache-dir-path.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- - - name: Set Yarn version - run: yarn policies set-version v1.22.22 - - name: Yarn install - uses: nick-fields/retry@v3 - env: - CUBESTORE_SKIP_POST_INSTALL: true - with: - max_attempts: 3 - retry_on: error - retry_wait_seconds: 15 - timeout_minutes: 20 - command: yarn install --frozen-lockfile - - name: Build Core Client libraries - run: yarn build - - name: Build other packages - run: yarn lerna run --concurrency 1 build - env: - NODE_OPTIONS: --max_old_space_size=4096 - - name: Lerna tsc - run: yarn tsc - - name: Build cubejs-backend-native (without Python) - run: yarn run native:build-release - working-directory: ./packages/cubejs-backend-native - - name: Run Integration tests for ${{ matrix.db }} matrix - uses: nick-fields/retry@v3 - with: - max_attempts: 3 - retry_on: error - retry_wait_seconds: 15 - timeout_minutes: 30 - command: ./.github/actions/integration/${{ matrix.db }}.sh - env: - CUBEJS_DB_BQ_CREDENTIALS: ${{ secrets.CUBEJS_DB_BQ_CREDENTIALS }} - CUBEJS_AWS_KEY: ${{ secrets.CUBEJS_AWS_KEY }} - CUBEJS_AWS_SECRET: ${{ secrets.CUBEJS_AWS_SECRET }} - CUBEJS_DB_USER: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_USER }} - CUBEJS_DB_PASS: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_PASS }} diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 1677171440784..ca907bfe5236c 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -4,6 +4,7 @@ on: push: paths: - '.github/actions/smoke.sh' + - '.github/actions/integration/**' - '.github/workflows/push.yml' - '.github/workflows/master.yml' - 'packages/**' @@ -23,6 +24,8 @@ on: - 'master' pull_request: paths: + - '.github/actions/smoke.sh' + - '.github/actions/integration/**' - '.github/workflows/push.yml' - '.github/workflows/master.yml' - 'packages/**' @@ -389,6 +392,9 @@ jobs: if: (needs['latest-tag-sha'].outputs.sha != github.sha) env: CLOUD_DATABASES: > + athena + bigquery + snowflake firebolt dremio # Athena (just to check for secrets availability) @@ -398,6 +404,7 @@ jobs: matrix: node-version: [22.x] db: [ + 'athena', 'bigquery', 'snowflake', 'clickhouse', 'druid', 'elasticsearch', 'mssql', 'mysql', 'postgres', 'prestodb', 'mysql-aurora-serverless', 'crate', 'mongobi', 'firebolt', 'dremio', 'vertica' ] @@ -449,6 +456,8 @@ jobs: retry_wait_seconds: 15 timeout_minutes: 20 command: yarn install --frozen-lockfile + - name: Build Core Client libraries + run: yarn build - name: Lerna tsc run: yarn tsc - name: Run Integration tests for ${{ matrix.db }} matrix @@ -475,6 +484,17 @@ jobs: DRIVERS_TESTS_DREMIO_CUBEJS_DB_NAME: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_NAME }} DRIVERS_TESTS_DREMIO_CUBEJS_DB_DREMIO_AUTH_TOKEN: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_DREMIO_AUTH_TOKEN }} + # BigQuery + DRIVERS_TESTS_SNOWFLAKE_CUBEJS_DB_BQ_CREDENTIALS: ${{ secrets.CUBEJS_DB_BQ_CREDENTIALS }} + + # AWS Athena + DRIVERS_TESTS_SNOWFLAKE_CUBEJS_AWS_KEY: ${{ secrets.CUBEJS_AWS_KEY }} + DRIVERS_TESTS_SNOWFLAKE_CUBEJS_AWS_SECRET: ${{ secrets.CUBEJS_AWS_SECRET }} + + # Snowflake + DRIVERS_TESTS_SNOWFLAKE_CUBEJS_DB_USER: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_USER }} + DRIVERS_TESTS_SNOWFLAKE_CUBEJS_DB_PASS: ${{ secrets.DRIVERS_TESTS_CUBEJS_DB_SNOWFLAKE_PASS }} + integration-smoke: needs: [latest-tag-sha, build-cubestore, build-native-linux] runs-on: ubuntu-24.04