include

Documentation

Index

• Variables

Variables

var Composeyml = strings.TrimSpace(`
version: '2'

networks:
  airflow:
    driver: bridge

volumes:
  postgres_data:
    driver: local
  airflow_logs:
    driver: local

services:
  postgres:
    image: postgres:10.1-alpine
    restart: unless-stopped
    networks:
      - airflow
    labels:
      io.astronomer.docker: "true"
      io.astronomer.docker.cli: "true"
    ports:
      - {{ .PostgresPort }}:5432
    volumes:
      - postgres_data:/var/lib/postgresql/data
    environment:
      POSTGRES_USER: {{ .PostgresUser }}
      POSTGRES_PASSWORD: {{ .PostgresPassword }}

  scheduler:
    image: {{ .AirflowImage }}
    command: ["airflow", "scheduler"]
    restart: unless-stopped
    networks:
      - airflow
    user: {{ .AirflowUser }}
    labels:
      io.astronomer.docker: "true"
      io.astronomer.docker.cli: "true"
      io.astronomer.docker.component: "airflow-scheduler"
    depends_on:
      - postgres
    environment:
      AIRFLOW__CORE__EXECUTOR: LocalExecutor
      AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql://{{ .PostgresUser }}:{{ .PostgresPassword }}@{{ .PostgresHost }}:5432
      AIRFLOW__CORE__LOAD_EXAMPLES: "False"
      AIRFLOW__CORE__FERNET_KEY: "d6Vefz3G9U_ynXB3cr7y_Ak35tAHkEGAVxuz_B-jzWw="
    volumes:
      - {{ .AirflowHome }}/dags:/usr/local/airflow/dags:ro
      - {{ .AirflowHome }}/plugins:/usr/local/airflow/plugins:{{ .MountLabel }}
      - {{ .AirflowHome }}/include:/usr/local/airflow/include:{{ .MountLabel }}
      - airflow_logs:/usr/local/airflow/logs
    {{ .AirflowEnvFile }}

  webserver:
    image: {{ .AirflowImage }}
    command: >
      bash -c "airflow create_user -r Admin -u admin -e admin@example.com -f admin -l user -p admin && airflow webserver"
    restart: unless-stopped
    networks:
      - airflow
    user: {{ .AirflowUser }}
    labels:
      io.astronomer.docker: "true"
      io.astronomer.docker.cli: "true"
      io.astronomer.docker.component: "airflow-webserver"
    depends_on:
      - scheduler
      - postgres
    environment:
      AIRFLOW__CORE__EXECUTOR: LocalExecutor
      AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql://{{ .PostgresUser }}:{{ .PostgresPassword }}@{{ .PostgresHost }}:5432
      AIRFLOW__CORE__LOAD_EXAMPLES: "False"
      AIRFLOW__CORE__FERNET_KEY: "d6Vefz3G9U_ynXB3cr7y_Ak35tAHkEGAVxuz_B-jzWw="
      AIRFLOW__WEBSERVER__RBAC: "True"
    ports:
      - {{ .AirflowWebserverPort }}:8080
    volumes:
      - {{ .AirflowHome }}/dags:/usr/local/airflow/dags:{{ .MountLabel }}
      - {{ .AirflowHome }}/plugins:/usr/local/airflow/plugins:{{ .MountLabel }}
      - {{ .AirflowHome }}/include:/usr/local/airflow/include:{{ .MountLabel }}
      - airflow_logs:/usr/local/airflow/logs
    {{ .AirflowEnvFile }}
`)

Composeyml is the docker-compose template

var Dockerfile = strings.TrimSpace(`
FROM astronomerinc/ap-airflow:%s
`)

Dockerfile is the Dockerfile template

var Dockerignore = strings.TrimSpace(`
.astro
.git
.env
airflow_settings.yaml
`)

Dockerignore is the .dockerignore template

var ExamplePlugin = strings.TrimSpace(`
from airflow.plugins_manager import AirflowPlugin
from flask_admin.base import MenuLink

"""
Look for the Astronomer tab in the UI.
"""
airflow_plugins_ml = MenuLink(
    category='Astronomer',
    name='Airflow-Plugins',
    url='https://github.com/airflow-plugins/')

astro_docs_ml = MenuLink(
    category='Astronomer',
    name='Astronomer Docs',
    url='https://www.astronomer.io/docs/')

astro_guides_ml = MenuLink(
    category='Astronomer',
    name='Airflow Guides',
    url='https://www.astronomer.io/guides/')


class AstroLinksPlugin(AirflowPlugin):
    name = 'astronomer_menu_links'
    operators = []
    flask_blueprints = []
    hooks = []
    executors = []
    macros = []
    admin_views = []
    menu_links = [airflow_plugins_ml, astro_docs_ml, astro_guides_ml]
		appbuilder_views = []
    appbuilder_menu_items = [
        {
            "name": ml.name,
            "category": ml.category,
            "category_icon": "fa-rocket",
            "href": ml.url,
        } for ml in menu_links
    ]`)

ExamplePlugin created with astro airflow init

var Exampledag = strings.TrimSpace(`
from airflow import DAG
from airflow.operators.bash_operator import BashOperator
from datetime import datetime, timedelta

default_args = {
	'owner': 'airflow',
	'depends_on_past': False,
	'start_date': datetime(2018, 1, 1),
	'email_on_failure': False,
	'email_on_retry': False,
	'retries': 1,
	'retry_delay': timedelta(minutes=5),
}

dag = DAG('example_dag',
			max_active_runs=3,
			schedule_interval=timedelta(minutes=5),
			default_args=default_args)

t1 = BashOperator(
	task_id='print_date1',
	bash_command='sleep $[ ( $RANDOM % 30 )  + 1 ]s',
	dag=dag)

t2 = BashOperator(
	task_id='print_date2',
	bash_command='sleep $[ ( $RANDOM % 30 )  + 1 ]s',
	dag=dag)

t3 = BashOperator(
	task_id='print_date3',
	bash_command='sleep $[ ( $RANDOM % 30 )  + 1 ]s',
	dag=dag)

t4 = BashOperator(
	task_id='print_date4',
	bash_command='sleep $[ ( $RANDOM % 30 )  + 1 ]s',
	dag=dag)

t5 = BashOperator(
	task_id='print_date5',
	bash_command='sleep $[ ( $RANDOM % 30 )  + 1 ]s',
	dag=dag)

t6 = BashOperator(
	task_id='print_date6',
	bash_command='sleep $[ ( $RANDOM % 30 )  + 1 ]s',
	dag=dag)

t7 = BashOperator(
	task_id='print_date7',
	bash_command='sleep $[ ( $RANDOM % 30 )  + 1 ]s',
	dag=dag)

t8 = BashOperator(
	task_id='print_date8',
	bash_command='sleep $[ ( $RANDOM % 30 )  + 1 ]s',
	dag=dag)

t2.set_upstream(t1)
t3.set_upstream(t2)
t4.set_upstream(t3)
t5.set_upstream(t3)
t6.set_upstream(t3)
t7.set_upstream(t3)
t8.set_upstream(t3)

t9 = BashOperator(
	task_id='print_date9',
	bash_command='sleep $[ ( $RANDOM % 30 )  + 1 ]s',
	dag=dag)

t10 = BashOperator(
	task_id='print_date10',
	bash_command='sleep $[ ( $RANDOM % 30 )  + 1 ]s',
	dag=dag)

t11 = BashOperator(
	task_id='print_date11',
	bash_command='sleep $[ ( $RANDOM % 30 )  + 1 ]s',
	dag=dag)

t12 = BashOperator(
	task_id='print_date12',
	bash_command='sleep $[ ( $RANDOM % 30 )  + 1 ]s',
	dag=dag)

t9.set_upstream(t8)
t10.set_upstream(t8)
t11.set_upstream(t8)
t12.set_upstream(t8)
`)

Exampledag created with astro airflow init

var Gitignore = strings.TrimSpace(`
.git
.env
airflow_settings.yaml
`)

Gitignore is the .gitignore template

var Settingsyml = strings.TrimSpace(`
# This feature is in Beta.

# Please report any bugs to https://github.com/astronomer/astro-cli/issues

# NOTE: If putting a dict in conn_extra, please wrap in single quotes.

# More details you can find https://github.com/astronomer/docs/blob/master/v0.10/cli-airflow-configuration.md

airflow:
  connections:
    - conn_id:
      conn_type:
      conn_host:
      conn_login:
      conn_password:
      conn_port:
      conn_extra:
  pools:
    - pool_name:
      pool_slot:
      pool_description:
  variables:
    - variable_name:
      variable_value:`)

Settingsyml is the settings template