feat(AIR302): extend the following rules
Any class in Airflow that inherits these class should not have these methods * `airflow.secrets.base_secrets.BaseSecretsBackend.get_conn_uri` → `airflow.secrets.base_secrets.BaseSecretsBackend.get_conn_value` * `airflow.secrets.base_secrets.BaseSecretsBackend.get_connections` → `airflow.secrets.base_secrets.BaseSecretsBackend.get_connection` * `airflow.hooks.base.BaseHook.get_connections` → use `get_connection` * `airflow.datasets.BaseDataset.iter_datasets` → `airflow.sdk.definitions.asset.BaseAsset.iter_assets` * `airflow.datasets.BaseDataset.iter_dataset_aliases` → `airflow.sdk.definitions.asset.BaseAsset.iter_asset_aliases`
This commit is contained in:
@@ -96,6 +96,10 @@ from airflow.www.utils import get_sensitive_variables_fields, should_hide_value_
|
||||
PY36, PY37, PY38, PY39, PY310, PY311, PY312
|
||||
DatasetFromRoot
|
||||
|
||||
dataset_from_root = DatasetFromRoot()
|
||||
dataset_from_root.iter_datasets()
|
||||
dataset_from_root.iter_dataset_aliases()
|
||||
|
||||
# airflow.api_connexion.security
|
||||
requires_access, requires_access_dataset
|
||||
|
||||
@@ -119,6 +123,18 @@ DatasetAny
|
||||
expand_alias_to_datasets
|
||||
Metadata
|
||||
|
||||
dataset_to_test_method_call = Dataset()
|
||||
dataset_to_test_method_call.iter_datasets()
|
||||
dataset_to_test_method_call.iter_dataset_aliases()
|
||||
|
||||
alias_to_test_method_call = DatasetAlias()
|
||||
alias_to_test_method_call.iter_datasets()
|
||||
alias_to_test_method_call.iter_dataset_aliases()
|
||||
|
||||
any_to_test_method_call = DatasetAny()
|
||||
any_to_test_method_call.iter_datasets()
|
||||
any_to_test_method_call.iter_dataset_aliases()
|
||||
|
||||
# airflow.datasets.manager
|
||||
DatasetManager, dataset_manager, resolve_dataset_manager
|
||||
|
||||
@@ -254,6 +270,9 @@ has_access_dataset
|
||||
# airflow.www.utils
|
||||
get_sensitive_variables_fields, should_hide_value_for_key
|
||||
|
||||
|
||||
# methods
|
||||
|
||||
from airflow.datasets.manager import DatasetManager
|
||||
|
||||
dm = DatasetManager()
|
||||
@@ -272,15 +291,34 @@ hlc.add_input_dataset()
|
||||
hlc.add_output_dataset()
|
||||
hlc.collected_datasets()
|
||||
|
||||
|
||||
from airflow.providers.amazon.auth_manager.aws_auth_manager import AwsAuthManager
|
||||
|
||||
aam = AwsAuthManager()
|
||||
aam.is_authorized_dataset()
|
||||
|
||||
|
||||
from airflow.providers_manager import ProvidersManager
|
||||
|
||||
pm = ProvidersManager()
|
||||
pm.initialize_providers_asset_uri_resources()
|
||||
pm.dataset_factories
|
||||
|
||||
|
||||
from airflow.secrets.base_secrets import BaseSecretsBackend
|
||||
|
||||
base_secret_backend = BaseSecretsBackend()
|
||||
base_secret_backend.get_conn_uri()
|
||||
base_secret_backend.get_connections()
|
||||
|
||||
from airflow.providers.google.cloud.secrets.secret_manager import (
|
||||
CloudSecretManagerBackend,
|
||||
)
|
||||
|
||||
csm_backend = CloudSecretManagerBackend()
|
||||
csm_backend.get_conn_uri()
|
||||
csm_backend.get_connections()
|
||||
|
||||
from airflow.providers.hashicorp.secrets.vault import VaultBackend
|
||||
|
||||
vault_backend = VaultBackend()
|
||||
vault_backend.get_conn_uri()
|
||||
vault_backend.get_connections()
|
||||
|
||||
@@ -4,8 +4,16 @@ use ruff_python_ast::{name::QualifiedName, Arguments, Expr, ExprAttribute, ExprC
|
||||
use ruff_python_semantic::analyze::typing;
|
||||
use ruff_python_semantic::Modules;
|
||||
use ruff_text_size::Ranged;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use crate::checkers::ast::Checker;
|
||||
use regex::Regex;
|
||||
|
||||
static SECRET_BACKEND_REGEX: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"airflow\..*secrets\.\w+\.\w+Backend").unwrap());
|
||||
|
||||
static AIRFLOW_HOOK_REGEX: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"airflow\..*hooks\.\w+\.\w+Hook").unwrap());
|
||||
|
||||
#[derive(Debug, Eq, PartialEq)]
|
||||
enum Replacement {
|
||||
@@ -196,6 +204,20 @@ fn removed_method(checker: &mut Checker, expr: &Expr) {
|
||||
)),
|
||||
&_ => None,
|
||||
},
|
||||
["airflow", "datasets", ..] | ["airflow", "Dataset"] => match attr.as_str() {
|
||||
"iter_datasets" => Some(Replacement::Name("iter_assets")),
|
||||
"iter_dataset_aliases" => Some(Replacement::Name("iter_asset_aliases")),
|
||||
&_ => None,
|
||||
},
|
||||
_ if SECRET_BACKEND_REGEX.is_match(&qualname.segments().join(".")) => match attr.as_str() {
|
||||
"get_conn_uri" => Some(Replacement::Name("get_conn_value")),
|
||||
"get_connections" => Some(Replacement::Name("get_connection")),
|
||||
&_ => None,
|
||||
},
|
||||
_ if AIRFLOW_HOOK_REGEX.is_match(&qualname.segments().join(".")) => match attr.as_str() {
|
||||
"get_connections" => Some(Replacement::Name("get_connection")),
|
||||
&_ => None,
|
||||
},
|
||||
_ => None,
|
||||
};
|
||||
if let Some(replacement) = replacement {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user