Documentation ¶
Index ¶
Constants ¶
This section is empty.
Variables ¶
View Source
var Composeyml = strings.TrimSpace(`
version: '2'
networks:
airflow:
driver: bridge
volumes:
postgres_data:
driver: local
airflow_logs:
driver: local
services:
postgres:
image: postgres:10.1-alpine
restart: unless-stopped
networks:
- airflow
labels:
io.astronomer.docker: "true"
io.astronomer.docker.cli: "true"
ports:
- {{ .PostgresPort }}:5432
volumes:
- postgres_data:/var/lib/postgresql/data
environment:
POSTGRES_USER: {{ .PostgresUser }}
POSTGRES_PASSWORD: {{ .PostgresPassword }}
scheduler:
image: {{ .AirflowImage }}
command: ["airflow", "scheduler"]
restart: unless-stopped
networks:
- airflow
user: {{ .AirflowUser }}
labels:
io.astronomer.docker: "true"
io.astronomer.docker.cli: "true"
io.astronomer.docker.component: "airflow-scheduler"
depends_on:
- postgres
environment:
AIRFLOW__CORE__EXECUTOR: LocalExecutor
AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql://{{ .PostgresUser }}:{{ .PostgresPassword }}@{{ .PostgresHost }}:5432
AIRFLOW__CORE__LOAD_EXAMPLES: "False"
AIRFLOW__CORE__FERNET_KEY: "d6Vefz3G9U_ynXB3cr7y_Ak35tAHkEGAVxuz_B-jzWw="
volumes:
- {{ .AirflowHome }}/dags:/usr/local/airflow/dags:ro
- {{ .AirflowHome }}/plugins:/usr/local/airflow/plugins:ro
- {{ .AirflowHome }}/include:/usr/local/airflow/include:ro
- airflow_logs:/usr/local/airflow/logs
{{ .AirflowEnvFile }}
webserver:
image: {{ .AirflowImage }}
command: >
bash -c "airflow create_user -r Admin -u admin -e admin@example.com -f admin -l user -p admin && airflow webserver"
restart: unless-stopped
networks:
- airflow
user: {{ .AirflowUser }}
labels:
io.astronomer.docker: "true"
io.astronomer.docker.cli: "true"
io.astronomer.docker.component: "airflow-webserver"
depends_on:
- scheduler
- postgres
environment:
AIRFLOW__CORE__EXECUTOR: LocalExecutor
AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql://{{ .PostgresUser }}:{{ .PostgresPassword }}@{{ .PostgresHost }}:5432
AIRFLOW__CORE__LOAD_EXAMPLES: "False"
AIRFLOW__CORE__FERNET_KEY: "d6Vefz3G9U_ynXB3cr7y_Ak35tAHkEGAVxuz_B-jzWw="
AIRFLOW__WEBSERVER__RBAC: "True"
ports:
- {{ .AirflowWebserverPort }}:8080
volumes:
- {{ .AirflowHome }}/dags:/usr/local/airflow/dags:ro
- {{ .AirflowHome }}/plugins:/usr/local/airflow/plugins:ro
- {{ .AirflowHome }}/include:/usr/local/airflow/include:ro
- airflow_logs:/usr/local/airflow/logs
{{ .AirflowEnvFile }}
`)
Composeyml is the docker-compose template
View Source
var Dockerfile = strings.TrimSpace(`
FROM astronomerinc/ap-airflow:%s
`)
Dockerfile is the Dockerfile template
View Source
var Dockerignore = strings.TrimSpace(`
.astro
.git
.env
airflow_setttings.yaml
`)
Dockerignore is the .dockerignore template
View Source
var ExamplePlugin = strings.TrimSpace(`
from airflow.plugins_manager import AirflowPlugin
from flask_admin.base import MenuLink
"""
Look for the Astronomer tab in the UI.
"""
airflow_plugins_ml = MenuLink(
category='Astronomer',
name='Airflow-Plugins',
url='https://github.com/airflow-plugins/')
astro_docs_ml = MenuLink(
category='Astronomer',
name='Astronomer Docs',
url='https://www.astronomer.io/docs/')
astro_guides_ml = MenuLink(
category='Astronomer',
name='Airflow Guides',
url='https://www.astronomer.io/guides/')
class AstroLinksPlugin(AirflowPlugin):
name = 'astronomer_menu_links'
operators = []
flask_blueprints = []
hooks = []
executors = []
macros = []
admin_views = []
menu_links = [airflow_plugins_ml, astro_docs_ml, astro_guides_ml]`)
ExamplePlugin created with astro airflow init
View Source
var Exampledag = strings.TrimSpace(`
from airflow import DAG
from airflow.operators.bash_operator import BashOperator
from datetime import datetime, timedelta
default_args = {
'owner': 'airflow',
'depends_on_past': False,
'start_date': datetime(2018, 1, 1),
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': timedelta(minutes=5),
}
dag = DAG('example_dag',
max_active_runs=3,
schedule_interval=timedelta(minutes=5),
default_args=default_args)
t1 = BashOperator(
task_id='print_date1',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s',
dag=dag)
t2 = BashOperator(
task_id='print_date2',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s',
dag=dag)
t3 = BashOperator(
task_id='print_date3',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s',
dag=dag)
t4 = BashOperator(
task_id='print_date4',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s',
dag=dag)
t5 = BashOperator(
task_id='print_date5',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s',
dag=dag)
t6 = BashOperator(
task_id='print_date6',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s',
dag=dag)
t7 = BashOperator(
task_id='print_date7',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s',
dag=dag)
t8 = BashOperator(
task_id='print_date8',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s',
dag=dag)
t2.set_upstream(t1)
t3.set_upstream(t2)
t4.set_upstream(t3)
t5.set_upstream(t3)
t6.set_upstream(t3)
t7.set_upstream(t3)
t8.set_upstream(t3)
t9 = BashOperator(
task_id='print_date9',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s',
dag=dag)
t10 = BashOperator(
task_id='print_date10',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s',
dag=dag)
t11 = BashOperator(
task_id='print_date11',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s',
dag=dag)
t12 = BashOperator(
task_id='print_date12',
bash_command='sleep $[ ( $RANDOM % 30 ) + 1 ]s',
dag=dag)
t9.set_upstream(t8)
t10.set_upstream(t8)
t11.set_upstream(t8)
t12.set_upstream(t8)
`)
Exampledag created with astro airflow init
View Source
var Gitignore = strings.TrimSpace(`
.astro
.git
.env
airflow_setttings.yaml
`)
Gitignore is the .gitignore template
View Source
var Settingsyml = strings.TrimSpace(`
# This feature is in Beta.
# Please report any bugs to https://github.com/astronomer/astro-cli/issues
# NOTE: If putting a dict in conn_extra, please wrap in single quotes.
# More details you can find https://github.com/astronomer/docs/blob/master/v0.10/cli-airflow-configuration.md
airflow:
connections:
- conn_id:
conn_type:
conn_host:
conn_login:
conn_password:
conn_port:
conn_extra:
pools:
- pool_name:
pool_slot:
pool_description:
variables:
- variable_name:
variable_value:`)
Settingsyml is the settings template
Functions ¶
This section is empty.
Types ¶
This section is empty.
Click to show internal directories.
Click to hide internal directories.