composer-demo-app/hub.yaml

version: 1
kind: stack
meta:
  name: Dataflow example with Composer Environment Sandbox

components:
  - name: network
    source:
      dir: components/network
      git:
        remote: https://github.com/agilestacks/google-components.git
        subDir: network
  - name: composer
    depends:
      - network
    source:
      dir: components/composer-environment
      git:
        remote: https://github.com/agilestacks/google-components.git
        subDir: composer-environment
  - name: example
    depends:
      - network
    source:
      dir: components/dataflow-example
      git:
        remote: https://github.com/agilestacks/google-components.git
        subDir: dataflow-example

requires:
  - gcp

extensions:
  init:
    - gcp
    - .hub/after-init
    - .hub/get-info
  configure:
    - gcp
    - env

lifecycle:
  order:
    - network
    - composer
    - example

parameters:
  - name: hub.sandboxDir
    fromEnv: SANDBOX_DIR
  - name: hub.sandboxCommit
    fromEnv: SANDBOX_COMMIT
  - name: hub.userAccount
    fromEnv: USER_ACCOUNT

  - name: dns.domain
    fromEnv: HUB_DOMAIN_NAME
  - name: dns.name
    fromEnv: HUB_STACK_NAME
  - name: stateBucketName
    fromEnv: HUB_STATE_BUCKET

  - name: cloud.region
    fromEnv: GOOGLE_REGION
  - name: cloud.zone
    fromEnv: GOOGLE_ZONE
  - name: projectId
    fromEnv: GOOGLE_PROJECT

  - name: component.network
    parameters:
      - name: name
        value: ${dns.name}
      - name: autocreateSubnets
        value: false
      - name: subnetwork.cidr
        value: 10.127.0.0/20

  - name: gke.node
    parameters:
      - name: count
        value: 3
        brief: GKE cluster nodes count
      - name: machineType
        value: n1-standard-1
        brief: GKE cluster nodes machine type
      - name: diskSize
        value: 100
        brief: GKE cluster nodes disk size (in GB)

  - name: composer
    parameters:
      - name: envName
        value: ${dns.name}-test-composer
      - name: version
        default: v1
        fromEnv: COMPOSER_VERSION
        brief: Version of composer environment (v1 or v2)
      - name: python.version
        default: 3
        fromEnv: COMPOSER_PYTHON_VERSION
        brief: Python version
      - name: python.requirementsFile
        default: ./requirements.txt
        fromEnv: COMPOSER_REQUIREMENTS_TXT
        brief: |
          Path to the requirements.txt file for composer custom pypi packages
          Absolute path or relative to the composer-environment component directory
      - name: airflow.imageVersion
        empty: allow
        fromEnv: AIRFLOW_IMAGE_VERSION
        brief: |
          Airflow image version (empty for latest)
          See available: https://cloud.google.com/composer/docs/concepts/versioning/composer-versions
      - name: endpoint
        default: public
        brief: "Composer endpoint (public or private)"
        fromEnv: COMPOSER_ENDPOINT
      - name: projectId
        value: ${projectId}
      - name: network.subnetwork
        kind: link
        value: ${component.network.subnetwork}

  - name: dataflow
    parameters:
      - name: bigquery
        parameters:
        - name: datasetName
          value: ${dns.name}_average_weather
        - name: tableName
          value: average_weather
        - name: location
          value: ${cloud.region}
        - name: deleteContentsOnDestroy
          value: true
      - name: composer
        parameters:
        - name: envName
          value: ${composer.envName}
        - name: bucket
          kind: link
          value: ${composer.gcs.bucket}
        - name: version
          value: ${composer.version}

outputs:
  - name: composer.airflow.url
  - name: composer.gcs.bucket
  - name: kubernetes.gke.cluster