feat(AIR302): argument appbuilder is now removed in BaseAuthManager and its subclasses

This commit is contained in:
Wei Lee
2024-12-26 17:06:29 +09:00
parent 316126cf38
commit 7ca2d283c1
3 changed files with 209 additions and 151 deletions

View File

@@ -6,6 +6,7 @@ from airflow.operators.trigger_dagrun import TriggerDagRunOperator
from airflow.providers.amazon.aws.log.s3_task_handler import S3TaskHandler
from airflow.providers.apache.hdfs.log.hdfs_task_handler import HdfsTaskHandler
from airflow.providers.elasticsearch.log.es_task_handler import ElasticsearchTaskHandler
from airflow.providers.fab.auth_manager.fab_auth_manager import FabAuthManager
from airflow.providers.google.cloud.log.gcs_task_handler import GCSTaskHandler
from airflow.providers.standard.operators import datetime, trigger_dagrun
from airflow.providers.standard.sensors import weekday
@@ -51,7 +52,9 @@ def decorator_deprecated_operator_args():
trigger_dagrun_op = trigger_dagrun.TriggerDagRunOperator(
task_id="trigger_dagrun_op1", execution_date="2024-12-04"
)
trigger_dagrun_op2 = TriggerDagRunOperator(task_id="trigger_dagrun_op2", execution_date="2024-12-04")
trigger_dagrun_op2 = TriggerDagRunOperator(
task_id="trigger_dagrun_op2", execution_date="2024-12-04"
)
branch_dt_op = datetime.BranchDateTimeOperator(
task_id="branch_dt_op", use_task_execution_day=True, task_concurrency=5
@@ -62,10 +65,16 @@ def decorator_deprecated_operator_args():
sla=timedelta(seconds=10),
)
dof_task_sensor = weekday.DayOfWeekSensor(task_id="dof_task_sensor", use_task_execution_day=True)
dof_task_sensor2 = DayOfWeekSensor(task_id="dof_task_sensor2", use_task_execution_day=True)
dof_task_sensor = weekday.DayOfWeekSensor(
task_id="dof_task_sensor", use_task_execution_day=True
)
dof_task_sensor2 = DayOfWeekSensor(
task_id="dof_task_sensor2", use_task_execution_day=True
)
bdow_op = weekday.BranchDayOfWeekOperator(task_id="bdow_op", use_task_execution_day=True)
bdow_op = weekday.BranchDayOfWeekOperator(
task_id="bdow_op", use_task_execution_day=True
)
bdow_op2 = BranchDayOfWeekOperator(task_id="bdow_op2", use_task_execution_day=True)
trigger_dagrun_op >> trigger_dagrun_op2
@@ -79,3 +88,5 @@ S3TaskHandler(filename_template="/tmp/test")
HdfsTaskHandler(filename_template="/tmp/test")
ElasticsearchTaskHandler(filename_template="/tmp/test")
GCSTaskHandler(filename_template="/tmp/test")
FabAuthManager(None)

View File

@@ -35,6 +35,30 @@ fn is_airflow_secret_backend(segments: &[&str]) -> bool {
}
}
fn is_airflow_auth_manager(segments: &[&str]) -> bool {
match segments {
["airflow", "auth", "manager", rest @ ..] => {
if let Some(last_element) = rest.last() {
last_element.ends_with("AuthManager")
} else {
false
}
}
["airflow", "providers", rest @ ..] => {
if let (Some(pos), Some(last_element)) =
(rest.iter().position(|&s| s == "auth_manager"), rest.last())
{
pos + 1 < rest.len() && last_element.ends_with("AuthManager")
} else {
false
}
}
_ => false,
}
}
fn is_airflow_hook(qualname: &QualifiedName) -> bool {
match qualname.segments() {
["airflow", "hooks", rest @ ..] => {
@@ -217,6 +241,20 @@ fn removed_argument(checker: &mut Checker, qualname: &QualifiedName, arguments:
None::<&str>,
));
}
_ if is_airflow_auth_manager(qualname.segments()) => {
if !arguments.is_empty() {
checker.diagnostics.push(Diagnostic::new(
Airflow3Removal {
// deprecated: (*arguments).to_string(),
deprecated: "appbuilder".to_string(),
replacement: Replacement::Message(
"The constructor takes no parameter now.",
),
},
arguments.range(),
));
}
}
_ if is_airflow_task_handler(qualname.segments()) => {
checker.diagnostics.extend(diagnostic_for_argument(
arguments,

View File

@@ -2,242 +2,251 @@
source: crates/ruff_linter/src/rules/airflow/mod.rs
snapshot_kind: text
---
AIR302_args.py:17:39: AIR302 [*] `schedule_interval` is removed in Airflow 3.0
AIR302_args.py:18:39: AIR302 [*] `schedule_interval` is removed in Airflow 3.0
|
15 | DAG(dag_id="class_schedule", schedule="@hourly")
16 |
17 | DAG(dag_id="class_schedule_interval", schedule_interval="@hourly")
16 | DAG(dag_id="class_schedule", schedule="@hourly")
17 |
18 | DAG(dag_id="class_schedule_interval", schedule_interval="@hourly")
| ^^^^^^^^^^^^^^^^^ AIR302
18 |
19 | DAG(dag_id="class_timetable", timetable=NullTimetable())
19 |
20 | DAG(dag_id="class_timetable", timetable=NullTimetable())
|
= help: Use `schedule` instead
Safe fix
14 14 |
15 15 | DAG(dag_id="class_schedule", schedule="@hourly")
16 16 |
17 |-DAG(dag_id="class_schedule_interval", schedule_interval="@hourly")
17 |+DAG(dag_id="class_schedule_interval", schedule="@hourly")
18 18 |
19 19 | DAG(dag_id="class_timetable", timetable=NullTimetable())
20 20 |
15 15 |
16 16 | DAG(dag_id="class_schedule", schedule="@hourly")
17 17 |
18 |-DAG(dag_id="class_schedule_interval", schedule_interval="@hourly")
18 |+DAG(dag_id="class_schedule_interval", schedule="@hourly")
19 19 |
20 20 | DAG(dag_id="class_timetable", timetable=NullTimetable())
21 21 |
AIR302_args.py:19:31: AIR302 [*] `timetable` is removed in Airflow 3.0
AIR302_args.py:20:31: AIR302 [*] `timetable` is removed in Airflow 3.0
|
17 | DAG(dag_id="class_schedule_interval", schedule_interval="@hourly")
18 |
19 | DAG(dag_id="class_timetable", timetable=NullTimetable())
18 | DAG(dag_id="class_schedule_interval", schedule_interval="@hourly")
19 |
20 | DAG(dag_id="class_timetable", timetable=NullTimetable())
| ^^^^^^^^^ AIR302
|
= help: Use `schedule` instead
Safe fix
16 16 |
17 17 | DAG(dag_id="class_schedule_interval", schedule_interval="@hourly")
18 18 |
19 |-DAG(dag_id="class_timetable", timetable=NullTimetable())
19 |+DAG(dag_id="class_timetable", schedule=NullTimetable())
20 20 |
17 17 |
18 18 | DAG(dag_id="class_schedule_interval", schedule_interval="@hourly")
19 19 |
20 |-DAG(dag_id="class_timetable", timetable=NullTimetable())
20 |+DAG(dag_id="class_timetable", schedule=NullTimetable())
21 21 |
22 22 | def sla_callback(*arg, **kwargs):
22 22 |
23 23 | def sla_callback(*arg, **kwargs):
AIR302_args.py:26:34: AIR302 `sla_miss_callback` is removed in Airflow 3.0
AIR302_args.py:27:34: AIR302 `sla_miss_callback` is removed in Airflow 3.0
|
26 | DAG(dag_id="class_sla_callback", sla_miss_callback=sla_callback)
27 | DAG(dag_id="class_sla_callback", sla_miss_callback=sla_callback)
| ^^^^^^^^^^^^^^^^^ AIR302
|
AIR302_args.py:34:6: AIR302 [*] `schedule_interval` is removed in Airflow 3.0
AIR302_args.py:35:6: AIR302 [*] `schedule_interval` is removed in Airflow 3.0
|
34 | @dag(schedule_interval="0 * * * *")
35 | @dag(schedule_interval="0 * * * *")
| ^^^^^^^^^^^^^^^^^ AIR302
35 | def decorator_schedule_interval():
36 | pass
36 | def decorator_schedule_interval():
37 | pass
|
= help: Use `schedule` instead
Safe fix
31 31 | pass
32 32 |
32 32 | pass
33 33 |
34 |-@dag(schedule_interval="0 * * * *")
34 |+@dag(schedule="0 * * * *")
35 35 | def decorator_schedule_interval():
36 36 | pass
37 37 |
34 34 |
35 |-@dag(schedule_interval="0 * * * *")
35 |+@dag(schedule="0 * * * *")
36 36 | def decorator_schedule_interval():
37 37 | pass
38 38 |
AIR302_args.py:39:6: AIR302 [*] `timetable` is removed in Airflow 3.0
AIR302_args.py:40:6: AIR302 [*] `timetable` is removed in Airflow 3.0
|
39 | @dag(timetable=NullTimetable())
40 | @dag(timetable=NullTimetable())
| ^^^^^^^^^ AIR302
40 | def decorator_timetable():
41 | pass
41 | def decorator_timetable():
42 | pass
|
= help: Use `schedule` instead
Safe fix
36 36 | pass
37 37 |
37 37 | pass
38 38 |
39 |-@dag(timetable=NullTimetable())
39 |+@dag(schedule=NullTimetable())
40 40 | def decorator_timetable():
41 41 | pass
42 42 |
39 39 |
40 |-@dag(timetable=NullTimetable())
40 |+@dag(schedule=NullTimetable())
41 41 | def decorator_timetable():
42 42 | pass
43 43 |
AIR302_args.py:44:6: AIR302 `sla_miss_callback` is removed in Airflow 3.0
AIR302_args.py:45:6: AIR302 `sla_miss_callback` is removed in Airflow 3.0
|
44 | @dag(sla_miss_callback=sla_callback)
45 | @dag(sla_miss_callback=sla_callback)
| ^^^^^^^^^^^^^^^^^ AIR302
45 | def decorator_sla_callback():
46 | pass
46 | def decorator_sla_callback():
47 | pass
|
AIR302_args.py:52:39: AIR302 [*] `execution_date` is removed in Airflow 3.0
AIR302_args.py:53:39: AIR302 [*] `execution_date` is removed in Airflow 3.0
|
50 | def decorator_deprecated_operator_args():
51 | trigger_dagrun_op = trigger_dagrun.TriggerDagRunOperator(
52 | task_id="trigger_dagrun_op1", execution_date="2024-12-04"
51 | def decorator_deprecated_operator_args():
52 | trigger_dagrun_op = trigger_dagrun.TriggerDagRunOperator(
53 | task_id="trigger_dagrun_op1", execution_date="2024-12-04"
| ^^^^^^^^^^^^^^ AIR302
53 | )
54 | trigger_dagrun_op2 = TriggerDagRunOperator(task_id="trigger_dagrun_op2", execution_date="2024-12-04")
54 | )
55 | trigger_dagrun_op2 = TriggerDagRunOperator(
|
= help: Use `logical_date` instead
Safe fix
49 49 | @dag()
50 50 | def decorator_deprecated_operator_args():
51 51 | trigger_dagrun_op = trigger_dagrun.TriggerDagRunOperator(
52 |- task_id="trigger_dagrun_op1", execution_date="2024-12-04"
52 |+ task_id="trigger_dagrun_op1", logical_date="2024-12-04"
53 53 | )
54 54 | trigger_dagrun_op2 = TriggerDagRunOperator(task_id="trigger_dagrun_op2", execution_date="2024-12-04")
55 55 |
50 50 | @dag()
51 51 | def decorator_deprecated_operator_args():
52 52 | trigger_dagrun_op = trigger_dagrun.TriggerDagRunOperator(
53 |- task_id="trigger_dagrun_op1", execution_date="2024-12-04"
53 |+ task_id="trigger_dagrun_op1", logical_date="2024-12-04"
54 54 | )
55 55 | trigger_dagrun_op2 = TriggerDagRunOperator(
56 56 | task_id="trigger_dagrun_op2", execution_date="2024-12-04"
AIR302_args.py:54:78: AIR302 [*] `execution_date` is removed in Airflow 3.0
AIR302_args.py:56:39: AIR302 [*] `execution_date` is removed in Airflow 3.0
|
52 | task_id="trigger_dagrun_op1", execution_date="2024-12-04"
53 | )
54 | trigger_dagrun_op2 = TriggerDagRunOperator(task_id="trigger_dagrun_op2", execution_date="2024-12-04")
| ^^^^^^^^^^^^^^ AIR302
55 |
56 | branch_dt_op = datetime.BranchDateTimeOperator(
54 | )
55 | trigger_dagrun_op2 = TriggerDagRunOperator(
56 | task_id="trigger_dagrun_op2", execution_date="2024-12-04"
| ^^^^^^^^^^^^^^ AIR302
57 | )
|
= help: Use `logical_date` instead
Safe fix
51 51 | trigger_dagrun_op = trigger_dagrun.TriggerDagRunOperator(
52 52 | task_id="trigger_dagrun_op1", execution_date="2024-12-04"
53 53 | )
54 |- trigger_dagrun_op2 = TriggerDagRunOperator(task_id="trigger_dagrun_op2", execution_date="2024-12-04")
54 |+ trigger_dagrun_op2 = TriggerDagRunOperator(task_id="trigger_dagrun_op2", logical_date="2024-12-04")
55 55 |
56 56 | branch_dt_op = datetime.BranchDateTimeOperator(
57 57 | task_id="branch_dt_op", use_task_execution_day=True, task_concurrency=5
53 53 | task_id="trigger_dagrun_op1", execution_date="2024-12-04"
54 54 | )
55 55 | trigger_dagrun_op2 = TriggerDagRunOperator(
56 |- task_id="trigger_dagrun_op2", execution_date="2024-12-04"
56 |+ task_id="trigger_dagrun_op2", logical_date="2024-12-04"
57 57 | )
58 58 |
59 59 | branch_dt_op = datetime.BranchDateTimeOperator(
AIR302_args.py:57:33: AIR302 [*] `use_task_execution_day` is removed in Airflow 3.0
AIR302_args.py:60:33: AIR302 [*] `use_task_execution_day` is removed in Airflow 3.0
|
56 | branch_dt_op = datetime.BranchDateTimeOperator(
57 | task_id="branch_dt_op", use_task_execution_day=True, task_concurrency=5
59 | branch_dt_op = datetime.BranchDateTimeOperator(
60 | task_id="branch_dt_op", use_task_execution_day=True, task_concurrency=5
| ^^^^^^^^^^^^^^^^^^^^^^ AIR302
58 | )
59 | branch_dt_op2 = BranchDateTimeOperator(
61 | )
62 | branch_dt_op2 = BranchDateTimeOperator(
|
= help: Use `use_task_logical_date` instead
Safe fix
54 54 | trigger_dagrun_op2 = TriggerDagRunOperator(task_id="trigger_dagrun_op2", execution_date="2024-12-04")
55 55 |
56 56 | branch_dt_op = datetime.BranchDateTimeOperator(
57 |- task_id="branch_dt_op", use_task_execution_day=True, task_concurrency=5
57 |+ task_id="branch_dt_op", use_task_logical_date=True, task_concurrency=5
58 58 | )
59 59 | branch_dt_op2 = BranchDateTimeOperator(
60 60 | task_id="branch_dt_op2",
57 57 | )
58 58 |
59 59 | branch_dt_op = datetime.BranchDateTimeOperator(
60 |- task_id="branch_dt_op", use_task_execution_day=True, task_concurrency=5
60 |+ task_id="branch_dt_op", use_task_logical_date=True, task_concurrency=5
61 61 | )
62 62 | branch_dt_op2 = BranchDateTimeOperator(
63 63 | task_id="branch_dt_op2",
AIR302_args.py:57:62: AIR302 [*] `task_concurrency` is removed in Airflow 3.0
AIR302_args.py:60:62: AIR302 [*] `task_concurrency` is removed in Airflow 3.0
|
56 | branch_dt_op = datetime.BranchDateTimeOperator(
57 | task_id="branch_dt_op", use_task_execution_day=True, task_concurrency=5
59 | branch_dt_op = datetime.BranchDateTimeOperator(
60 | task_id="branch_dt_op", use_task_execution_day=True, task_concurrency=5
| ^^^^^^^^^^^^^^^^ AIR302
58 | )
59 | branch_dt_op2 = BranchDateTimeOperator(
61 | )
62 | branch_dt_op2 = BranchDateTimeOperator(
|
= help: Use `max_active_tis_per_dag` instead
Safe fix
54 54 | trigger_dagrun_op2 = TriggerDagRunOperator(task_id="trigger_dagrun_op2", execution_date="2024-12-04")
55 55 |
56 56 | branch_dt_op = datetime.BranchDateTimeOperator(
57 |- task_id="branch_dt_op", use_task_execution_day=True, task_concurrency=5
57 |+ task_id="branch_dt_op", use_task_execution_day=True, max_active_tis_per_dag=5
58 58 | )
59 59 | branch_dt_op2 = BranchDateTimeOperator(
60 60 | task_id="branch_dt_op2",
57 57 | )
58 58 |
59 59 | branch_dt_op = datetime.BranchDateTimeOperator(
60 |- task_id="branch_dt_op", use_task_execution_day=True, task_concurrency=5
60 |+ task_id="branch_dt_op", use_task_execution_day=True, max_active_tis_per_dag=5
61 61 | )
62 62 | branch_dt_op2 = BranchDateTimeOperator(
63 63 | task_id="branch_dt_op2",
AIR302_args.py:61:9: AIR302 [*] `use_task_execution_day` is removed in Airflow 3.0
AIR302_args.py:64:9: AIR302 [*] `use_task_execution_day` is removed in Airflow 3.0
|
59 | branch_dt_op2 = BranchDateTimeOperator(
60 | task_id="branch_dt_op2",
61 | use_task_execution_day=True,
62 | branch_dt_op2 = BranchDateTimeOperator(
63 | task_id="branch_dt_op2",
64 | use_task_execution_day=True,
| ^^^^^^^^^^^^^^^^^^^^^^ AIR302
62 | sla=timedelta(seconds=10),
63 | )
65 | sla=timedelta(seconds=10),
66 | )
|
= help: Use `use_task_logical_date` instead
Safe fix
58 58 | )
59 59 | branch_dt_op2 = BranchDateTimeOperator(
60 60 | task_id="branch_dt_op2",
61 |- use_task_execution_day=True,
61 |+ use_task_logical_date=True,
62 62 | sla=timedelta(seconds=10),
63 63 | )
64 64 |
61 61 | )
62 62 | branch_dt_op2 = BranchDateTimeOperator(
63 63 | task_id="branch_dt_op2",
64 |- use_task_execution_day=True,
64 |+ use_task_logical_date=True,
65 65 | sla=timedelta(seconds=10),
66 66 | )
67 67 |
AIR302_args.py:62:9: AIR302 `sla` is removed in Airflow 3.0
AIR302_args.py:65:9: AIR302 `sla` is removed in Airflow 3.0
|
60 | task_id="branch_dt_op2",
61 | use_task_execution_day=True,
62 | sla=timedelta(seconds=10),
63 | task_id="branch_dt_op2",
64 | use_task_execution_day=True,
65 | sla=timedelta(seconds=10),
| ^^^ AIR302
63 | )
66 | )
|
AIR302_args.py:78:15: AIR302 `filename_template` is removed in Airflow 3.0
AIR302_args.py:87:15: AIR302 `filename_template` is removed in Airflow 3.0
|
77 | # deprecated filename_template arugment in FileTaskHandler
78 | S3TaskHandler(filename_template="/tmp/test")
86 | # deprecated filename_template arugment in FileTaskHandler
87 | S3TaskHandler(filename_template="/tmp/test")
| ^^^^^^^^^^^^^^^^^ AIR302
79 | HdfsTaskHandler(filename_template="/tmp/test")
80 | ElasticsearchTaskHandler(filename_template="/tmp/test")
88 | HdfsTaskHandler(filename_template="/tmp/test")
89 | ElasticsearchTaskHandler(filename_template="/tmp/test")
|
AIR302_args.py:79:17: AIR302 `filename_template` is removed in Airflow 3.0
AIR302_args.py:88:17: AIR302 `filename_template` is removed in Airflow 3.0
|
77 | # deprecated filename_template arugment in FileTaskHandler
78 | S3TaskHandler(filename_template="/tmp/test")
79 | HdfsTaskHandler(filename_template="/tmp/test")
86 | # deprecated filename_template arugment in FileTaskHandler
87 | S3TaskHandler(filename_template="/tmp/test")
88 | HdfsTaskHandler(filename_template="/tmp/test")
| ^^^^^^^^^^^^^^^^^ AIR302
80 | ElasticsearchTaskHandler(filename_template="/tmp/test")
81 | GCSTaskHandler(filename_template="/tmp/test")
89 | ElasticsearchTaskHandler(filename_template="/tmp/test")
90 | GCSTaskHandler(filename_template="/tmp/test")
|
AIR302_args.py:80:26: AIR302 `filename_template` is removed in Airflow 3.0
AIR302_args.py:89:26: AIR302 `filename_template` is removed in Airflow 3.0
|
78 | S3TaskHandler(filename_template="/tmp/test")
79 | HdfsTaskHandler(filename_template="/tmp/test")
80 | ElasticsearchTaskHandler(filename_template="/tmp/test")
87 | S3TaskHandler(filename_template="/tmp/test")
88 | HdfsTaskHandler(filename_template="/tmp/test")
89 | ElasticsearchTaskHandler(filename_template="/tmp/test")
| ^^^^^^^^^^^^^^^^^ AIR302
81 | GCSTaskHandler(filename_template="/tmp/test")
90 | GCSTaskHandler(filename_template="/tmp/test")
|
AIR302_args.py:81:16: AIR302 `filename_template` is removed in Airflow 3.0
AIR302_args.py:90:16: AIR302 `filename_template` is removed in Airflow 3.0
|
79 | HdfsTaskHandler(filename_template="/tmp/test")
80 | ElasticsearchTaskHandler(filename_template="/tmp/test")
81 | GCSTaskHandler(filename_template="/tmp/test")
88 | HdfsTaskHandler(filename_template="/tmp/test")
89 | ElasticsearchTaskHandler(filename_template="/tmp/test")
90 | GCSTaskHandler(filename_template="/tmp/test")
| ^^^^^^^^^^^^^^^^^ AIR302
91 |
92 | FabAuthManager(None)
|
AIR302_args.py:92:15: AIR302 `appbuilder` is removed in Airflow 3.0; The constructor takes no parameter now.
|
90 | GCSTaskHandler(filename_template="/tmp/test")
91 |
92 | FabAuthManager(None)
| ^^^^^^ AIR302
|