通过修改来回答我自己的问题self.retries
实例变量,可用于所有运算符,在execute
方法我们可以动态地强制不再重试。
在以下示例中:
- 传感器 0:第一次尝试就会成功
- 传感器 1:4 次尝试后将失败(最多重试 1 + 3 次)
- 传感器 2:尝试 1 次后将失败(动态强制不再重试)
from datetime import datetime, timedelta
from airflow import DAG
from airflow.models import BaseOperator
class PseudoSensor(BaseOperator):
def __init__(
self,
s3_status_code_mock,
*args,
**kwargs):
super().__init__(*args, **kwargs)
self.s3_status_code_mock = s3_status_code_mock
def execute(self, context):
# Try to read S3, Redshift, blah blah
pass
# The query returned a status code, that we mock when the Sensor is initialized
if self.s3_status_code_mock == 0:
# Success
return 0
elif self.s3_status_code_mock == 1:
# Error but should retry if I can still can
raise Exception("Retryable error. Won't change retries of operator.")
elif self.s3_status_code_mock == 2:
# Unrecoverable error. Should fail without future retries.
self.retries = 0
raise Exception("Unrecoverable error. Will set retries to 0.")
# A separate function so we don't make the globals dirty
def createDAG():
# Default (but overridable) arguments for Operators instantiations
default_args = {
'owner': 'Supay',
'depends_on_past': False,
'start_date': datetime(2019, 11, 28),
'retry_delay': timedelta(seconds=1),
'retries': 3,
}
with DAG("dynamic_retries_dag", default_args=default_args, schedule_interval=timedelta(days=1), catchup=False) as dag :
# Sensor 0: should succeed in first try
sensor_0 = PseudoSensor(
task_id="sensor_0",
provide_context=True,
s3_status_code_mock=0,
)
# Sensor 1: should fail after 3 tries
sensor_1 = PseudoSensor(
task_id="sensor_1",
provide_context=True,
s3_status_code_mock=1
)
# Sensor 1: should fail after 1 try
sensor_2 = PseudoSensor(
task_id="sensor_2",
provide_context=True,
s3_status_code_mock=2
)
dag >> sensor_0
dag >> sensor_1
dag >> sensor_2
globals()[dag.dag_id] = dag
# Run everything
createDAG()
Gantt showing the tries per task