feat(AIR302): extend the following rules

Any class in Airflow that inherits these class should not have these methods

* `airflow.secrets.base_secrets.BaseSecretsBackend.get_conn_uri` → `airflow.secrets.base_secrets.BaseSecretsBackend.get_conn_value`
* `airflow.secrets.base_secrets.BaseSecretsBackend.get_connections` → `airflow.secrets.base_secrets.BaseSecretsBackend.get_connection`
* `airflow.hooks.base.BaseHook.get_connections` → use `get_connection`
* `airflow.datasets.BaseDataset.iter_datasets` → `airflow.sdk.definitions.asset.BaseAsset.iter_assets`
* `airflow.datasets.BaseDataset.iter_dataset_aliases` → `airflow.sdk.definitions.asset.BaseAsset.iter_asset_aliases`
This commit is contained in:
Wei Lee
2024-12-20 20:45:58 +09:00
parent 04d538113a
commit e292b7b277
3 changed files with 717 additions and 475 deletions

View File

@@ -96,6 +96,10 @@ from airflow.www.utils import get_sensitive_variables_fields, should_hide_value_
PY36, PY37, PY38, PY39, PY310, PY311, PY312
DatasetFromRoot
dataset_from_root = DatasetFromRoot()
dataset_from_root.iter_datasets()
dataset_from_root.iter_dataset_aliases()
# airflow.api_connexion.security
requires_access, requires_access_dataset
@@ -119,6 +123,18 @@ DatasetAny
expand_alias_to_datasets
Metadata
dataset_to_test_method_call = Dataset()
dataset_to_test_method_call.iter_datasets()
dataset_to_test_method_call.iter_dataset_aliases()
alias_to_test_method_call = DatasetAlias()
alias_to_test_method_call.iter_datasets()
alias_to_test_method_call.iter_dataset_aliases()
any_to_test_method_call = DatasetAny()
any_to_test_method_call.iter_datasets()
any_to_test_method_call.iter_dataset_aliases()
# airflow.datasets.manager
DatasetManager, dataset_manager, resolve_dataset_manager
@@ -254,6 +270,9 @@ has_access_dataset
# airflow.www.utils
get_sensitive_variables_fields, should_hide_value_for_key
# methods
from airflow.datasets.manager import DatasetManager
dm = DatasetManager()
@@ -272,15 +291,34 @@ hlc.add_input_dataset()
hlc.add_output_dataset()
hlc.collected_datasets()
from airflow.providers.amazon.auth_manager.aws_auth_manager import AwsAuthManager
aam = AwsAuthManager()
aam.is_authorized_dataset()
from airflow.providers_manager import ProvidersManager
pm = ProvidersManager()
pm.initialize_providers_asset_uri_resources()
pm.dataset_factories
from airflow.secrets.base_secrets import BaseSecretsBackend
base_secret_backend = BaseSecretsBackend()
base_secret_backend.get_conn_uri()
base_secret_backend.get_connections()
from airflow.providers.google.cloud.secrets.secret_manager import (
CloudSecretManagerBackend,
)
csm_backend = CloudSecretManagerBackend()
csm_backend.get_conn_uri()
csm_backend.get_connections()
from airflow.providers.hashicorp.secrets.vault import VaultBackend
vault_backend = VaultBackend()
vault_backend.get_conn_uri()
vault_backend.get_connections()

View File

@@ -4,8 +4,16 @@ use ruff_python_ast::{name::QualifiedName, Arguments, Expr, ExprAttribute, ExprC
use ruff_python_semantic::analyze::typing;
use ruff_python_semantic::Modules;
use ruff_text_size::Ranged;
use std::sync::LazyLock;
use crate::checkers::ast::Checker;
use regex::Regex;
static SECRET_BACKEND_REGEX: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"airflow\..*secrets\.\w+\.\w+Backend").unwrap());
static AIRFLOW_HOOK_REGEX: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"airflow\..*hooks\.\w+\.\w+Hook").unwrap());
#[derive(Debug, Eq, PartialEq)]
enum Replacement {
@@ -196,6 +204,20 @@ fn removed_method(checker: &mut Checker, expr: &Expr) {
)),
&_ => None,
},
["airflow", "datasets", ..] | ["airflow", "Dataset"] => match attr.as_str() {
"iter_datasets" => Some(Replacement::Name("iter_assets")),
"iter_dataset_aliases" => Some(Replacement::Name("iter_asset_aliases")),
&_ => None,
},
_ if SECRET_BACKEND_REGEX.is_match(&qualname.segments().join(".")) => match attr.as_str() {
"get_conn_uri" => Some(Replacement::Name("get_conn_value")),
"get_connections" => Some(Replacement::Name("get_connection")),
&_ => None,
},
_ if AIRFLOW_HOOK_REGEX.is_match(&qualname.segments().join(".")) => match attr.as_str() {
"get_connections" => Some(Replacement::Name("get_connection")),
&_ => None,
},
_ => None,
};
if let Some(replacement) = replacement {