Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[airflow]: extend removed names (AIR302) #14804

Merged
merged 10 commits into from
Dec 6, 2024
Merged

[airflow]: extend removed names (AIR302) #14804

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
28d5916
feat(AIR302-name): "chain", "cross_downstream" was removed from "airf…
Lee-W Dec 6, 2024
954896c
feat(AIR302-name): the following functions are removed in "airflow.co…
Lee-W Dec 6, 2024
c48faf1
feat(AIR302-name): "load_connections" and "get_connection" was rename…
Lee-W Dec 6, 2024
6650062
feat(AIR302-name): "get_sensitive_variables_fields" and "should_hide_…
Lee-W Dec 6, 2024
bb1341b
feat(AIR302-name): "airflow.contrib.*" was removed
Lee-W Dec 6, 2024
e9a2a55
feat(AIR302-name): "airflow.operators.subdag.*" was removed
Lee-W Dec 6, 2024
2fdd02e
feat(AIR302-name): multiple operators was removed from airflow.operators
Lee-W Dec 6, 2024
6610c8d
feat(AIR302-name): DUMMY and NONE_FAILED_OR_SKIPPED was removed in ai…
Lee-W Dec 6, 2024
08a6484
feat(AIR302-name): airflow.PY36, airflow.PY37, airflow.PY38, airflow.…
Lee-W Dec 6, 2024
98bf367
feat(AIR302-name): airflow.utils.decorators.apply_defaults was removed
Lee-W Dec 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 64 additions & 3 deletions crates/ruff_linter/resources/test/fixtures/airflow/AIR302_names.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,35 @@
from airflow import PY36, PY37, PY38, PY39, PY310, PY311, PY312
from airflow.triggers.external_task import TaskStateTrigger
from airflow.www.auth import has_access
from airflow.api_connexion.security import requires_access
from airflow.configuration import (
get,
getboolean,
getfloat,
getint,
has_option,
remove_option,
as_dict,
set,
)
from airflow.contrib.aws_athena_hook import AWSAthenaHook
from airflow.metrics.validators import AllowListValidator
from airflow.metrics.validators import BlockListValidator
from airflow.operators.subdag import SubDagOperator
from airflow.sensors.external_task import ExternalTaskSensorLink
from airflow.operators.bash_operator import BashOperator
from airflow.operators.branch_operator import BaseBranchOperator
from airflow.operators.dummy import EmptyOperator, DummyOperator
from airflow.operators import dummy_operator
from airflow.operators.email_operator import EmailOperator
from airflow.sensors.base_sensor_operator import BaseSensorOperator
from airflow.sensors.date_time_sensor import DateTimeSensor
from airflow.sensors.external_task_sensor import (
ExternalTaskMarker,
ExternalTaskSensor,
ExternalTaskSensorLink,
)
from airflow.sensors.time_delta_sensor import TimeDeltaSensor
from airflow.secrets.local_filesystem import get_connection, load_connections
from airflow.utils import dates
from airflow.utils.dates import (
date_range,
Expand All @@ -13,20 +40,27 @@
round_time,
scale_time_units,
)
from airflow.utils.decorators import apply_defaults
from airflow.utils.file import TemporaryDirectory, mkdirs
from airflow.utils.helpers import chain, cross_downstream
from airflow.utils.state import SHUTDOWN, terminating_states
from airflow.utils.dag_cycle_tester import test_cycle
from airflow.utils.trigger_rule import TriggerRule
from airflow.www.auth import has_access
from airflow.www.utils import get_sensitive_variables_fields, should_hide_value_for_key

PY36, PY37, PY38, PY39, PY310, PY311, PY312

AWSAthenaHook
TaskStateTrigger


has_access
requires_access

AllowListValidator
BlockListValidator

SubDagOperator

dates.date_range
dates.days_ago

Expand All @@ -42,11 +76,38 @@
datetime_to_nano
dates.datetime_to_nano

get, getboolean, getfloat, getint, has_option, remove_option, as_dict, set

get_connection, load_connections


ExternalTaskSensorLink
BashOperator
BaseBranchOperator
EmptyOperator, DummyOperator
dummy_operator.EmptyOperator
dummy_operator.DummyOperator
EmailOperator
BaseSensorOperator
DateTimeSensor
(ExternalTaskMarker, ExternalTaskSensor, ExternalTaskSensorLink)
TimeDeltaSensor

apply_defaults

TemporaryDirectory
mkdirs

chain
cross_downstream

SHUTDOWN
terminating_states

TriggerRule.DUMMY
TriggerRule.NONE_FAILED_OR_SKIPPED

test_cycle

has_access
get_sensitive_variables_fields, should_hide_value_for_key
196 changes: 189 additions & 7 deletions crates/ruff_linter/src/rules/airflow/rules/removal_in_3.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ use crate::checkers::ast::Checker;
enum Replacement {
None,
Name(String),
Message(String),
}

/// ## What it does
Expand Down Expand Up @@ -53,6 +54,9 @@ impl Violation for Airflow3Removal {
Replacement::Name(name) => {
format!("`{deprecated}` is removed in Airflow 3.0; use `{name}` instead")
}
Replacement::Message(message) => {
format!("`{deprecated}` is removed in Airflow 3.0; {message}")
}
}
}
}
Expand Down Expand Up @@ -106,16 +110,76 @@ fn removed_name(checker: &mut Checker, expr: &Expr, ranged: impl Ranged) {
["airflow", "triggers", "external_task", "TaskStateTrigger"] => {
Some((qualname.to_string(), Replacement::None))
}
["airflow", "www", "auth", "has_access"] => Some((
qualname.to_string(),
Replacement::Name("airflow.www.auth.has_access_*".to_string()),
)),
["airflow", "api_connexion", "security", "requires_access"] => Some((
qualname.to_string(),
Replacement::Name(
"airflow.api_connexion.security.requires_access_*".to_string(),
),
)),
// airflow.PY\d{1,2}
["airflow", "PY36"] => Some((
qualname.to_string(),
Replacement::Name("sys.version_info".to_string()),
)),
["airflow", "PY37"] => Some((
qualname.to_string(),
Replacement::Name("sys.version_info".to_string()),
)),
["airflow", "PY38"] => Some((
qualname.to_string(),
Replacement::Name("sys.version_info".to_string()),
)),
["airflow", "PY39"] => Some((
qualname.to_string(),
Replacement::Name("sys.version_info".to_string()),
)),
["airflow", "PY310"] => Some((
qualname.to_string(),
Replacement::Name("sys.version_info".to_string()),
)),
["airflow", "PY311"] => Some((
qualname.to_string(),
Replacement::Name("sys.version_info".to_string()),
)),
["airflow", "PY312"] => Some((
qualname.to_string(),
Replacement::Name("sys.version_info".to_string()),
)),
// airflow.configuration
["airflow", "configuration", "get"] => Some((
qualname.to_string(),
Replacement::Name("airflow.configuration.conf.get".to_string()),
)),
["airflow", "configuration", "getboolean"] => Some((
qualname.to_string(),
Replacement::Name("airflow.configuration.conf.getboolean".to_string()),
)),
["airflow", "configuration", "getfloat"] => Some((
qualname.to_string(),
Replacement::Name("airflow.configuration.conf.getfloat".to_string()),
)),
["airflow", "configuration", "getint"] => Some((
qualname.to_string(),
Replacement::Name("airflow.configuration.conf.getint".to_string()),
)),
["airflow", "configuration", "has_option"] => Some((
qualname.to_string(),
Replacement::Name("airflow.configuration.conf.has_option".to_string()),
)),
["airflow", "configuration", "remove_option"] => Some((
qualname.to_string(),
Replacement::Name("airflow.configuration.conf.remove_option".to_string()),
)),
["airflow", "configuration", "as_dict"] => Some((
qualname.to_string(),
Replacement::Name("airflow.configuration.conf.as_dict".to_string()),
)),
["airflow", "configuration", "set"] => Some((
qualname.to_string(),
Replacement::Name("airflow.configuration.conf.set".to_string()),
)),
// airflow.contrib.*
["airflow", "contrib", ..] => Some((qualname.to_string(), Replacement::None)),
// airflow.metrics.validators
["airflow", "metrics", "validators", "AllowListValidator"] => Some((
qualname.to_string(),
Expand All @@ -129,6 +193,85 @@ fn removed_name(checker: &mut Checker, expr: &Expr, ranged: impl Ranged) {
"airflow.metrics.validators.PatternBlockListValidator".to_string(),
),
)),
// airflow.operators
["airflow", "operators", "subdag", ..] => {
Some((qualname.to_string(), Replacement::None))
}
["airflow.sensors.external_task.ExternalTaskSensorLink"] => Some((
qualname.to_string(),
Replacement::Name("airflow.sensors.external_task.ExternalDagLin".to_string()),
)),
["airflow", "operators", "bash_operator", "BashOperator"] => Some((
qualname.to_string(),
Replacement::Name("airflow.operators.bash.BashOperator".to_string()),
)),
["airflow", "operators", "branch_operator", "BaseBranchOperator"] => Some((
qualname.to_string(),
Replacement::Name("airflow.operators.branch.BaseBranchOperator".to_string()),
)),
["airflow", "operators", " dummy", "EmptyOperator"] => Some((
qualname.to_string(),
Replacement::Name("airflow.operators.empty.EmptyOperator".to_string()),
)),
["airflow", "operators", "dummy", "DummyOperator"] => Some((
qualname.to_string(),
Replacement::Name("airflow.operators.empty.EmptyOperator".to_string()),
)),
["airflow", "operators", "dummy_operator", "EmptyOperator"] => Some((
qualname.to_string(),
Replacement::Name("airflow.operators.empty.EmptyOperator".to_string()),
)),
["airflow", "operators", "dummy_operator", "DummyOperator"] => Some((
qualname.to_string(),
Replacement::Name("airflow.operators.empty.EmptyOperator".to_string()),
)),
["airflow", "operators", "email_operator", "EmailOperator"] => Some((
qualname.to_string(),
Replacement::Name("airflow.operators.email.EmailOperator".to_string()),
)),
["airflow", "sensors", "base_sensor_operator", "BaseSensorOperator"] => Some((
qualname.to_string(),
Replacement::Name("airflow.sensors.base.BaseSensorOperator".to_string()),
)),
["airflow", "sensors", "date_time_sensor", "DateTimeSensor"] => Some((
qualname.to_string(),
Replacement::Name("airflow.sensors.date_time.DateTimeSensor".to_string()),
)),
["airflow", "sensors", "external_task_sensor", "ExternalTaskMarker"] => Some((
qualname.to_string(),
Replacement::Name(
"airflow.sensors.external_task.ExternalTaskMarker".to_string(),
),
)),
["airflow", "sensors", "external_task_sensor", "ExternalTaskSensor"] => Some((
qualname.to_string(),
Replacement::Name(
"airflow.sensors.external_task.ExternalTaskSensor".to_string(),
),
)),
["airflow", "sensors", "external_task_sensor", "ExternalTaskSensorLink"] => Some((
qualname.to_string(),
Replacement::Name(
"airflow.sensors.external_task.ExternalTaskSensorLink".to_string(),
),
)),
["airflow", "sensors", "time_delta_sensor", "TimeDeltaSensor"] => Some((
qualname.to_string(),
Replacement::Name("airflow.sensors.time_delta.TimeDeltaSensor".to_string()),
)),
// airflow.secrets
["airflow", "secrets", "local_filesystem", "load_connections"] => Some((
qualname.to_string(),
Replacement::Name(
"airflow.secrets.local_filesystem.load_connections_dict".to_string(),
),
)),
["airflow", "secrets", "local_filesystem", "get_connection"] => Some((
qualname.to_string(),
Replacement::Name(
"airflow.secrets.local_filesystem.load_connections_dict".to_string(),
),
)),
// airflow.utils.dates
["airflow", "utils", "dates", "date_range"] => Some((
qualname.to_string(),
Expand Down Expand Up @@ -158,20 +301,59 @@ fn removed_name(checker: &mut Checker, expr: &Expr, ranged: impl Ranged) {
qualname.to_string(),
Replacement::Name("pendulum.today('UTC').add(days=-N, ...)".to_string()),
)),

// airflow.utils.helpers
["airflow", "utils", "helpers", "chain"] => Some((
qualname.to_string(),
Replacement::Name("airflow.models.baseoperator.chain".to_string()),
)),
["airflow", "utils", "helpers", "cross_downstream"] => Some((
qualname.to_string(),
Replacement::Name("airflow.models.baseoperator.cross_downstream".to_string()),
)),
// airflow.utils.state
["airflow", "utils", "state", "SHUTDOWN"] => {
Some((qualname.to_string(), Replacement::None))
}
["airflow", "utils", "state", "terminating_states"] => {
Some((qualname.to_string(), Replacement::None))
}
// airflow.uilts
["airflow", "utils", "dag_cycle_tester", "test_cycle"] => {
// airflow.utils.trigger_rule
["airflow", "utils", "trigger_rule", "TriggerRule", "DUMMY"] => {
Some((qualname.to_string(), Replacement::None))
}
["airflow", "utils", "decorators", "apply_defaults"] => {
["airflow", "utils", "trigger_rule", "TriggerRule", "NONE_FAILED_OR_SKIPPED"] => {
Some((qualname.to_string(), Replacement::None))
}
// airflow.uilts
["airflow", "utils", "dag_cycle_tester", "test_cycle"] => {
Some((qualname.to_string(), Replacement::None))
}
["airflow", "utils", "decorators", "apply_defaults"] => Some((
qualname.to_string(),
Replacement::Message(
"`apply_defaults` is now unconditionally done and can be safely removed."
.to_string(),
),
)),
// airflow.www
["airflow", "www", "auth", "has_access"] => Some((
qualname.to_string(),
Replacement::Name("airflow.www.auth.has_access_*".to_string()),
)),
["airflow", "www", "utils", "get_sensitive_variables_fields"] => Some((
qualname.to_string(),
Replacement::Name(
"airflow.utils.log.secrets_masker.get_sensitive_variables_fields"
.to_string(),
),
)),
["airflow", "www", "utils", "should_hide_value_for_key"] => Some((
qualname.to_string(),
Replacement::Name(
"airflow.utils.log.secrets_masker.should_hide_value_for_key".to_string(),
),
)),
_ => None,
});
if let Some((deprecated, replacement)) = result {
Expand Down
Loading
Loading