From bc6c4bf59d2d79b6f60a29f03e0f1e6ccf17254e Mon Sep 17 00:00:00 2001 From: Steve Purcell Date: Fri, 21 Aug 2026 12:28:46 +0100 Subject: [PATCH] Conform to the newer Ruff rules, and use the LocalStack defaults --- .github/workflows/wiremock.yml | 2 +- keycloak/localstack_keycloak/extension.py | 6 ++-- keycloak/sample-app/cdk/stacks/api_stack.py | 16 +++++++--- .../sample-app/lambda/authorizer/handler.py | 2 -- keycloak/tests/test_extension.py | 13 +++++--- paradedb/README.md | 2 +- paradedb/localstack_paradedb/extension.py | 4 +-- paradedb/tests/test_extension.py | 3 +- ruff.toml | 18 +++++++++++ typedb/tests/test_extension.py | 10 ++++-- utils/localstack_extensions/utils/__init__.py | 8 ++--- utils/localstack_extensions/utils/docker.py | 31 ++++++++++--------- utils/localstack_extensions/utils/h2_proxy.py | 16 +++++----- .../utils/tcp_protocol_router.py | 12 +++---- utils/tests/integration/conftest.py | 5 ++- .../integration/test_extension_integration.py | 2 +- utils/tests/integration/test_http2_proxy.py | 5 ++- utils/tests/unit/test_h2_frame_parsing.py | 6 ++-- wiremock/localstack_wiremock/extension.py | 7 +++-- wiremock/sample-app-oss/src/handler.py | 1 + 20 files changed, 99 insertions(+), 70 deletions(-) create mode 100644 ruff.toml diff --git a/.github/workflows/wiremock.yml b/.github/workflows/wiremock.yml index a8cb329d..f31722e4 100644 --- a/.github/workflows/wiremock.yml +++ b/.github/workflows/wiremock.yml @@ -51,7 +51,7 @@ jobs: docker pull "$IMAGE_NAME" docker pull wiremock/wiremock docker pull public.ecr.aws/lambda/python:3.12 - pip install localstack terraform-local awscli-local[ver1] + pip install localstack terraform-local awscli-local make install make lint diff --git a/keycloak/localstack_keycloak/extension.py b/keycloak/localstack_keycloak/extension.py index 27bc2e9b..df0c5e5c 100644 --- a/keycloak/localstack_keycloak/extension.py +++ b/keycloak/localstack_keycloak/extension.py @@ -9,7 +9,10 @@ from localstack_extensions.utils.docker import ProxiedDockerContainerExtension from .utils import ( + DEFAULT_AUDIENCE, DEFAULT_CLIENT_SECRET, + DEFAULT_REALM, + DEFAULT_VERSION, ENV_KEYCLOAK_DEFAULT_PASSWORD, ENV_KEYCLOAK_DEFAULT_USER, ENV_KEYCLOAK_FLAGS, @@ -19,9 +22,6 @@ ENV_KEYCLOAK_VERSION, KEYCLOAK_HTTP_PORT, KEYCLOAK_MGMT_PORT, - DEFAULT_AUDIENCE, - DEFAULT_REALM, - DEFAULT_VERSION, get_default_client_config, get_default_realm_config, get_env, diff --git a/keycloak/sample-app/cdk/stacks/api_stack.py b/keycloak/sample-app/cdk/stacks/api_stack.py index 854b832e..a9a79394 100644 --- a/keycloak/sample-app/cdk/stacks/api_stack.py +++ b/keycloak/sample-app/cdk/stacks/api_stack.py @@ -1,14 +1,20 @@ +import os + from aws_cdk import ( - Stack, Duration, - aws_lambda as lambda_, + RemovalPolicy, + Stack, +) +from aws_cdk import ( aws_apigateway as apigw, +) +from aws_cdk import ( aws_dynamodb as dynamodb, - aws_iam as iam, - RemovalPolicy, +) +from aws_cdk import ( + aws_lambda as lambda_, ) from constructs import Construct -import os class KeycloakSampleApiStack(Stack): diff --git a/keycloak/sample-app/lambda/authorizer/handler.py b/keycloak/sample-app/lambda/authorizer/handler.py index f09ae673..b07bfa40 100644 --- a/keycloak/sample-app/lambda/authorizer/handler.py +++ b/keycloak/sample-app/lambda/authorizer/handler.py @@ -10,8 +10,6 @@ """ import base64 -import hashlib -import hmac import json import os import urllib.request diff --git a/keycloak/tests/test_extension.py b/keycloak/tests/test_extension.py index f6a4c23d..f6b5d8fe 100644 --- a/keycloak/tests/test_extension.py +++ b/keycloak/tests/test_extension.py @@ -1,12 +1,12 @@ -import json import base64 +import json import time + +import boto3 import pytest import requests -import boto3 from botocore.config import Config - LOCALSTACK_URL = "http://localhost:4566" KEYCLOAK_URL = "http://keycloak.localhost.localstack.cloud:4566" KEYCLOAK_DIRECT_URL = "http://localhost:8080" # Direct access to Keycloak HTTP port @@ -189,7 +189,7 @@ class TestUserManagement: def test_create_user_with_required_profile_fields(self, admin_token): """Verify user creation works with required Keycloak 26+ profile fields. - + Keycloak 26+ requires email, firstName, lastName for users to be "fully set up". Password must be set separately via reset-password endpoint. """ @@ -302,7 +302,10 @@ def test_incomplete_user_fails_password_grant(self, admin_token): timeout=30, ) assert token_response.status_code != 200 - assert "not fully set up" in token_response.json().get("error_description", "").lower() + assert ( + "not fully set up" + in token_response.json().get("error_description", "").lower() + ) # Cleanup requests.delete(f"{users_url}/{user_id}", headers=headers, timeout=10) diff --git a/paradedb/README.md b/paradedb/README.md index 46163b63..5776bfb0 100644 --- a/paradedb/README.md +++ b/paradedb/README.md @@ -31,7 +31,7 @@ conn = psycopg2.connect( port=5432, database="mydatabase", user="myuser", - password="mypassword" + password="mypassword", ) ``` diff --git a/paradedb/localstack_paradedb/extension.py b/paradedb/localstack_paradedb/extension.py index a666bdbf..305fa06e 100644 --- a/paradedb/localstack_paradedb/extension.py +++ b/paradedb/localstack_paradedb/extension.py @@ -1,8 +1,8 @@ import os import socket -from localstack_extensions.utils.docker import ProxiedDockerContainerExtension from localstack import config +from localstack_extensions.utils.docker import ProxiedDockerContainerExtension # Environment variables for configuration ENV_POSTGRES_USER = "PARADEDB_POSTGRES_USER" @@ -88,7 +88,7 @@ def _check_tcp_port(self, host: str, port: int, timeout: float = 2.0) -> None: try: sock.connect((host, port)) sock.close() - except (socket.timeout, socket.error) as e: + except (TimeoutError, OSError) as e: raise AssertionError(f"Port {port} not ready: {e}") def get_connection_info(self) -> dict: diff --git a/paradedb/tests/test_extension.py b/paradedb/tests/test_extension.py index 7151488d..e77f35d9 100644 --- a/paradedb/tests/test_extension.py +++ b/paradedb/tests/test_extension.py @@ -3,7 +3,6 @@ from localstack.utils.strings import short_uid from localstack.utils.sync import retry - # Connection details for ParadeDB # Connect through LocalStack gateway with TCP proxying HOST = "paradedb.localhost.localstack.cloud" @@ -15,6 +14,7 @@ def get_connection(): """Create a connection to ParadeDB, retrying until the server is ready.""" + def _connect(): return psycopg2.connect( host=HOST, @@ -23,6 +23,7 @@ def _connect(): password=PASSWORD, database=DATABASE, ) + return retry(_connect, retries=15, sleep=2.0) diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000..c7fff6bc --- /dev/null +++ b/ruff.toml @@ -0,0 +1,18 @@ +[lint] +ignore = [ + "B006", # TODO Do not use mutable data structures for argument defaults + "B007", # TODO Loop control variable x not used within loop body + "B017", # TODO `pytest.raises(Exception)` should be considered evil + "B019", # TODO Use of `functools.lru_cache` or `functools.cache` on methods can lead to memory leaks + "B023", # TODO Function definition does not bind loop variable `server` + "B024", # TODO x is an abstract base class, but it has no abstract methods + "B026", # TODO Star-arg unpacking after a keyword argument is strongly discouraged + "B027", # TODO `Server.do_shutdown` is an empty method in an abstract base class, but has no abstract decorator + "B904", # TODO Within an `except` clause, raise exceptions with `raise ... from err` or `raise ... from None` to distinguish them from errors in exception handling + "C901", # TODO function is too complex + "E402", # TODO Module level import not at top of file + "E501", # E501 Line too long - handled by black, see https://docs.astral.sh/ruff/faq/#is-ruff-compatible-with-black + "E741", # TODO Ambiguous variable name: `l` + "T201", # TODO `print` found +] +select = ["B", "C", "E", "F", "I", "W", "T", "B9", "G", "UP", "TID251"] diff --git a/typedb/tests/test_extension.py b/typedb/tests/test_extension.py index dafd549c..24fc9e23 100644 --- a/typedb/tests/test_extension.py +++ b/typedb/tests/test_extension.py @@ -1,7 +1,13 @@ -import requests import httpx +import requests from localstack.utils.strings import short_uid -from typedb.driver import TypeDB, Credentials, DriverOptions, TransactionType, DriverTlsConfig +from typedb.driver import ( + Credentials, + DriverOptions, + DriverTlsConfig, + TransactionType, + TypeDB, +) def test_connect_to_db_via_http_api(): diff --git a/utils/localstack_extensions/utils/__init__.py b/utils/localstack_extensions/utils/__init__.py index 42a1e623..58d89153 100644 --- a/utils/localstack_extensions/utils/__init__.py +++ b/utils/localstack_extensions/utils/__init__.py @@ -3,21 +3,21 @@ ProxyResource, ) from localstack_extensions.utils.h2_proxy import ( + ProxyRequestMatcher, TcpForwarder, apply_http2_patches_for_grpc_support, + get_frames_from_http2_stream, get_headers_from_data_stream, get_headers_from_frames, - get_frames_from_http2_stream, - ProxyRequestMatcher, ) __all__ = [ "ProxiedDockerContainerExtension", + "ProxyRequestMatcher", "ProxyResource", "TcpForwarder", "apply_http2_patches_for_grpc_support", + "get_frames_from_http2_stream", "get_headers_from_data_stream", "get_headers_from_frames", - "get_frames_from_http2_stream", - "ProxyRequestMatcher", ] diff --git a/utils/localstack_extensions/utils/docker.py b/utils/localstack_extensions/utils/docker.py index de625723..070bccce 100644 --- a/utils/localstack_extensions/utils/docker.py +++ b/utils/localstack_extensions/utils/docker.py @@ -1,20 +1,17 @@ -import re import logging +import re +from collections.abc import Callable from functools import cache -from typing import Callable -import requests +import requests from localstack.config import is_env_true -from localstack_extensions.utils.h2_proxy import ( - apply_http2_patches_for_grpc_support, -) -from localstack.utils.docker_utils import DOCKER_CLIENT from localstack.extensions.api import Extension, http from localstack.http import Request from localstack.utils.container_utils.container_client import ( PortMappings, SimpleVolumeBind, ) +from localstack.utils.docker_utils import DOCKER_CLIENT from localstack.utils.net import get_addressable_container_host from localstack.utils.sync import retry from rolo import route @@ -22,6 +19,10 @@ from rolo.routing import RuleAdapter, WithHost from werkzeug.datastructures import Headers +from localstack_extensions.utils.h2_proxy import ( + apply_http2_patches_for_grpc_support, +) + LOG = logging.getLogger(__name__) @@ -104,7 +105,10 @@ def __init__( tcp_ports: list[int] | None = None, ): try: - from localstack.pro.core.utils.container.registry_strategies import CustomizableRegistryStrategy + from localstack.pro.core.utils.container.registry_strategies import ( + CustomizableRegistryStrategy, + ) + self.image_name = CustomizableRegistryStrategy().resolve(image_name) except ImportError: self.image_name = image_name @@ -138,10 +142,7 @@ def update_gateway_routes(self, router: http.Router[http.RouteHandler]): # Determine if HTTP proxy should be set up. Skip it when all container ports are # TCP-only and no host restriction is set, since a catch-all HTTP proxy would # intercept all requests and break other services. - uses_http = ( - self.host - and set(self.container_ports) - set(self.tcp_ports or []) - ) + uses_http = self.host and set(self.container_ports) - set(self.tcp_ports or []) if uses_http: # add resource for HTTP/1.1 requests @@ -179,8 +180,8 @@ def _setup_tcp_protocol_routing(self): matcher = getattr(self, "tcp_connection_matcher", None) if not matcher: LOG.warning( - f"Extension {self.name} has tcp_ports but no tcp_connection_matcher(). " - "TCP routing will not work without a matcher." + "Extension %s has tcp_ports but no tcp_connection_matcher(). " + "TCP routing will not work without a matcher.", self.name ) return @@ -199,7 +200,7 @@ def _setup_tcp_protocol_routing(self): ) LOG.info( - f"Registered TCP extension {self.name} -> {self.container_host}:{target_port} on gateway" + "Registered TCP extension %s -> %s:%s on gateway", self.name, self.container_host, target_port ) def http2_request_matcher(self, headers: Headers) -> bool: diff --git a/utils/localstack_extensions/utils/h2_proxy.py b/utils/localstack_extensions/utils/h2_proxy.py index 84ed0cb0..e4f990e0 100644 --- a/utils/localstack_extensions/utils/h2_proxy.py +++ b/utils/localstack_extensions/utils/h2_proxy.py @@ -1,14 +1,13 @@ import logging import socket +from collections.abc import Callable, Iterable from enum import Enum -from typing import Iterable, Callable from h2.frame_buffer import FrameBuffer from hpack import Decoder -from hyperframe.frame import HeadersFrame, Frame -from twisted.internet import reactor - +from hyperframe.frame import Frame, HeadersFrame from localstack.utils.patch import patch +from twisted.internet import reactor from twisted.web._http2 import H2Connection from werkzeug.datastructures import Headers @@ -50,7 +49,7 @@ def send(self, data): def close(self): if self._closed: return - LOG.debug(f"Closing connection to upstream HTTP2 server on port {self.port}") + LOG.debug("Closing connection to upstream HTTP2 server on port %d", self.port) self._closed = True try: self._socket.shutdown(socket.SHUT_RDWR) @@ -70,7 +69,7 @@ def apply_http2_patches_for_grpc_support( Apply some patches to proxy incoming gRPC requests and forward them to a target port. Note: this is a very brute-force approach and needs to be fixed/enhanced over time! """ - LOG.debug(f"Enabling proxying to backend {target_host}:{target_port}") + LOG.debug("Enabling proxying to backend %s:%s", target_host, target_port) global patched_connection assert not patched_connection, ( "It is not safe to patch H2Connection twice with this function" @@ -96,7 +95,7 @@ class ForwardingBuffer: def __init__(self, http_response_stream): self.http_response_stream = http_response_stream LOG.debug( - f"Starting TCP forwarder to port {target_port} for new HTTP2 connection" + "Starting TCP forwarder to port %s for new HTTP2 connection", target_port ) self.backend = TcpForwarder(target_port, host=target_host) self.buffer = [] @@ -175,7 +174,6 @@ def get_frames_from_http2_stream(data: bytes) -> Iterable[Frame]: buffer.max_frame_size = 16384 try: buffer.add_data(data) - for frame in buffer: - yield frame + yield from buffer except Exception: pass diff --git a/utils/localstack_extensions/utils/tcp_protocol_router.py b/utils/localstack_extensions/utils/tcp_protocol_router.py index bb3045b1..24280fc8 100644 --- a/utils/localstack_extensions/utils/tcp_protocol_router.py +++ b/utils/localstack_extensions/utils/tcp_protocol_router.py @@ -7,13 +7,13 @@ """ import logging + +from localstack import config +from localstack.utils.patch import patch from twisted.internet import reactor from twisted.protocols.portforward import ProxyClient, ProxyClientFactory from twisted.web.http import HTTPChannel -from localstack.utils.patch import patch -from localstack import config - LOG = logging.getLogger(__name__) LOG.setLevel(logging.DEBUG if config.DEBUG else logging.INFO) @@ -117,7 +117,7 @@ def _patched_dataReceived(fn, self, data): reactor.connectTCP(backend_host, backend_port, client_factory) return except Exception as e: - LOG.debug(f"Error in matcher for {ext_name}: {e}") + LOG.debug("Error in matcher for %s: %s", ext_name, e) continue # No extension claimed the connection @@ -164,7 +164,7 @@ def register_tcp_extension( """ _tcp_extensions.append((extension_name, matcher, backend_host, backend_port)) LOG.info( - f"Registered TCP extension {extension_name} -> {backend_host}:{backend_port}" + "Registered TCP extension %s -> %s:%s", extension_name, backend_host, backend_port ) @@ -176,4 +176,4 @@ def unregister_tcp_extension(extension_name: str): for name, matcher, host, port in _tcp_extensions if name != extension_name ] - LOG.info(f"Unregistered TCP extension {extension_name}") + LOG.info("Unregistered TCP extension %s", extension_name) diff --git a/utils/tests/integration/conftest.py b/utils/tests/integration/conftest.py index 6702c5c9..aa6f0d87 100644 --- a/utils/tests/integration/conftest.py +++ b/utils/tests/integration/conftest.py @@ -15,13 +15,12 @@ import pytest from hyperframe.frame import Frame from localstack.utils.net import get_free_tcp_port +from localstack_extensions.utils.docker import ProxiedDockerContainerExtension from rolo import Router from rolo.gateway import Gateway from twisted.internet import reactor from twisted.web import server as twisted_server -from localstack_extensions.utils.docker import ProxiedDockerContainerExtension - GRPCBIN_IMAGE = "moul/grpcbin" GRPCBIN_INSECURE_PORT = 9000 # HTTP/2 without TLS GRPCBIN_SECURE_PORT = 9001 # HTTP/2 with TLS @@ -52,7 +51,7 @@ def _tcp_health_check(): # Use container_host from the parent class sock.connect((self.container_host, GRPCBIN_INSECURE_PORT)) sock.close() - except (socket.timeout, socket.error) as e: + except (TimeoutError, OSError) as e: raise AssertionError(f"Port {GRPCBIN_INSECURE_PORT} not ready: {e}") super().__init__( diff --git a/utils/tests/integration/test_extension_integration.py b/utils/tests/integration/test_extension_integration.py index 56d8c060..6ef28aac 100644 --- a/utils/tests/integration/test_extension_integration.py +++ b/utils/tests/integration/test_extension_integration.py @@ -38,7 +38,7 @@ def test_extension_ports_are_reachable(self, grpcbin_extension_server): sock.connect(("localhost", gateway_port)) sock.close() # Connection successful - except (socket.timeout, socket.error) as e: + except (TimeoutError, OSError) as e: raise AssertionError(f"Could not connect to gateway port: {e}") def test_extension_implements_required_methods(self, grpcbin_extension): diff --git a/utils/tests/integration/test_http2_proxy.py b/utils/tests/integration/test_http2_proxy.py index a6ba6003..1556a29b 100644 --- a/utils/tests/integration/test_http2_proxy.py +++ b/utils/tests/integration/test_http2_proxy.py @@ -7,13 +7,12 @@ """ import threading -import pytest +import pytest from hyperframe.frame import SettingsFrame - from localstack_extensions.utils.h2_proxy import ( - get_headers_from_frames, TcpForwarder, + get_headers_from_frames, ) # Import from conftest - pytest automatically loads conftest.py diff --git a/utils/tests/unit/test_h2_frame_parsing.py b/utils/tests/unit/test_h2_frame_parsing.py index 6d7e3b14..4fc9a41c 100644 --- a/utils/tests/unit/test_h2_frame_parsing.py +++ b/utils/tests/unit/test_h2_frame_parsing.py @@ -6,15 +6,13 @@ No Docker or network access required. """ -from hyperframe.frame import SettingsFrame, HeadersFrame, WindowUpdateFrame - +from hyperframe.frame import HeadersFrame, SettingsFrame, WindowUpdateFrame from localstack_extensions.utils.h2_proxy import ( get_frames_from_http2_stream, - get_headers_from_frames, get_headers_from_data_stream, + get_headers_from_frames, ) - # HTTP/2 connection preface (24 bytes) HTTP2_PREFACE = b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" diff --git a/wiremock/localstack_wiremock/extension.py b/wiremock/localstack_wiremock/extension.py index 032782ef..20008ce8 100644 --- a/wiremock/localstack_wiremock/extension.py +++ b/wiremock/localstack_wiremock/extension.py @@ -1,13 +1,12 @@ import logging import os from pathlib import Path -import requests +import requests from localstack import config, constants from localstack.utils.net import get_addressable_container_host from localstack_extensions.utils.docker import ProxiedDockerContainerExtension - LOG = logging.getLogger(__name__) # If set, uses wiremock-runner image; otherwise uses OSS wiremock image @@ -45,7 +44,9 @@ def __init__(self): env_vars["WMC_ADMIN_PORT"] = str(ADMIN_PORT) env_vars["WMC_API_TOKEN"] = api_token env_vars["WMC_RUNNER_ENABLED"] = "true" - image_name = os.getenv(ENV_WIREMOCK_IMAGE_RUNNER) or self.DOCKER_IMAGE_RUNNER + image_name = ( + os.getenv(ENV_WIREMOCK_IMAGE_RUNNER) or self.DOCKER_IMAGE_RUNNER + ) container_ports = [SERVICE_PORT, ADMIN_PORT] health_check_path = "/__/health" health_check_retries = 90 diff --git a/wiremock/sample-app-oss/src/handler.py b/wiremock/sample-app-oss/src/handler.py index fc31bb10..f12bec34 100644 --- a/wiremock/sample-app-oss/src/handler.py +++ b/wiremock/sample-app-oss/src/handler.py @@ -1,4 +1,5 @@ import json + import requests