From c1c14ca6f02e41b3eac32f764759bc32c2bd47a0 Mon Sep 17 00:00:00 2001 From: Ruslan Dautkhanov Date: Thu, 21 May 2026 11:42:03 -0600 Subject: [PATCH 1/4] docs: align Python version claims to 3.9-3.13 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Python 2 and 3.8 support was already removed in #585; this catches up the public-facing claims: * root ``setup.py``: drop ``3.8`` classifier, add ``3.13`` (was missing from the root metadata — py4j-python/setup.py already had it). Add ``python_requires=">=3.9"`` so pip refuses to install on EOL interpreters. * ``py4j-web/install.rst``: section header + body to "3.9-3.13"; Windows path example uses generic ``python3X`` (was ``python27``). * ``py4j-web/download.rst``: "tested with 3.8+" -> "3.9+". * ``py4j-web/contributing.rst`` and root ``CONTRIBUTING.rst``: the baseline-compatibility line goes from 3.8 to 3.9. Note: the README ``shields.io/pypi/pyversions/py4j.svg`` badge reads from PyPI's classifiers — it'll auto-update to the corrected list on the next package publish; no README edit needed here. Co-authored-by: Isaac --- CONTRIBUTING.rst | 2 +- py4j-web/contributing.rst | 2 +- py4j-web/download.rst | 2 +- py4j-web/install.rst | 10 +++++----- setup.py | 3 ++- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index aa3c4711..e2907f6c 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -61,7 +61,7 @@ We follow pep8 rather stricly: 3. Line length is 80 4. Code must pass the default flake8 (version 2.5) tests (pep8 + pyflakes) -Code must be compatible with Python 3.8 to the newest released +Code must be compatible with Python 3.9 to the newest released version of Python. If external libraries must be used, they should be wrapped in a mechanism that diff --git a/py4j-web/contributing.rst b/py4j-web/contributing.rst index 709e4900..d6117340 100644 --- a/py4j-web/contributing.rst +++ b/py4j-web/contributing.rst @@ -43,7 +43,7 @@ We follow pep8 rather strictly: 3. Line length is 80. 4. Code must pass the default flake8 tests (pep8 + pyflakes). -Code must be compatible with Python 3.8 to the newest released +Code must be compatible with Python 3.9 to the newest released version of Python. If external libraries must be used, they should be wrapped in a mechanism that diff --git a/py4j-web/download.rst b/py4j-web/download.rst index 03136090..4d67427d 100644 --- a/py4j-web/download.rst +++ b/py4j-web/download.rst @@ -32,7 +32,7 @@ Requirements Py4J requires: -* A Python interpreter. Py4J has been tested with 3.8+. +* A Python interpreter. Py4J has been tested with 3.9+. * Java 7.0+. Py4J for Eclipse requires: diff --git a/py4j-web/install.rst b/py4j-web/install.rst index e82f7dce..1ddcd51e 100644 --- a/py4j-web/install.rst +++ b/py4j-web/install.rst @@ -3,12 +3,12 @@ Installing Py4J =============== -Installing Python 3.8-3.12 +Installing Python 3.9-3.13 --------------------------------- Py4J is a library written in Python and Java. Currently, Py4J has been tested -with Python 3.8, 3.9, 3.10, 3.11 and 3.12. You can install Python by going to the -`official Python download page `_. +with Python 3.9, 3.10, 3.11, 3.12 and 3.13. You can install Python by going to +the `official Python download page `_. Installing Java 7+ @@ -42,8 +42,8 @@ Using easy_install or pip 1. Either ``/usr/share/py4j/py4j0.x.jar`` or ``/usr/local/share/py4j/py4j0.x.jar`` for system-wide install on Linux. 2. ``{virtual_env_dir}/share/py4j/py4j0.x.jar`` for installation in a virtual environment. - 3. ``C:\python27\share\py4j\py4j0.x.jar`` for system-wide install on - Windows. + 3. ``C:\python3X\share\py4j\py4j0.x.jar`` for system-wide install on + Windows (where ``3X`` is your Python minor version). Using an official release ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/setup.py b/setup.py index 555ea398..e8c2fc21 100644 --- a/setup.py +++ b/setup.py @@ -48,17 +48,18 @@ author="Barthelemy Dagenais", author_email="barthelemy@infobart.com", license="BSD License", + python_requires=">=3.9", classifiers=[ "Development Status :: 4 - Beta", "Intended Audience :: Developers", "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", "Programming Language :: Python", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Java", "Topic :: Software Development :: Libraries", "Topic :: Software Development :: Object Brokering", From 4dea614d0c86cf0be6031082b8fc1748a3c21280 Mon Sep 17 00:00:00 2001 From: Ruslan Dautkhanov Date: Thu, 21 May 2026 11:47:39 -0600 Subject: [PATCH 2/4] style: modernize Python 3 syntax (f-strings, super(), while True, exception chaining) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mechanical Python 3 syntax modernization across py4j-python/src/py4j/. No behavior change; all 53 non-JVM unit tests pass, full matrix is the final validation. Categories: * **.format() -> f-strings** (38 conversions): java_gateway.py (21), java_collections.py (10), clientserver.py (5), protocol.py (2). Conservative — sites with named placeholders, format specs ({0:5d}), conversion flags ({0!r}), or escape braces ({{ }}) are deliberately left as .format(). The remaining ~4 .format() calls are correct as-is. * **%-style formatting -> f-strings** (3 conversions): clientserver.py socket-handshake address/port writes. Also collapsed "z\n".encode("utf-8") to b"z\n" while in the neighborhood. * **super(Cls, self) -> super()** (15 conversions): clientserver.py (9), java_gateway.py (3), protocol.py (3 exception classes). Py3-only syntax. * **while(True) -> while True** (1 site in clientserver.py): PEP 8 / Py3-only style. * **raise X from e** (3 sites): java_gateway.py GatewayConnection.send_command, _get_connection, and CallbackServer.start_callback; clientserver.py ClientServerConnection.send_command. Preserves the original cause in tracebacks — improves debug context for PySpark users (and anyone else) without changing the public API. Sites NOT converted (deliberately): .format() with conversion flags ({0!r}), escape braces ({{x}}), or multi-line continuations — the regex converter was tuned conservative to avoid the subtle bugs that caused the failure on @markjm's #575 review. Co-authored-by: Isaac --- py4j-python/src/py4j/clientserver.py | 43 ++++++++-------- py4j-python/src/py4j/java_collections.py | 26 ++++------ py4j-python/src/py4j/java_gateway.py | 62 +++++++++++------------- py4j-python/src/py4j/protocol.py | 11 ++--- 4 files changed, 63 insertions(+), 79 deletions(-) diff --git a/py4j-python/src/py4j/clientserver.py b/py4j-python/src/py4j/clientserver.py index 88a31c21..225be498 100644 --- a/py4j-python/src/py4j/clientserver.py +++ b/py4j-python/src/py4j/clientserver.py @@ -39,10 +39,10 @@ class FinalizerWorker(Thread): def __init__(self, deque): self.deque = deque - super(FinalizerWorker, self).__init__() + super().__init__() def run(self): - while(True): + while True: try: task = self.deque.pop() if task == SHUTDOWN_FINALIZER_WORKER: @@ -119,7 +119,7 @@ def __init__( :param auth_token: if provided, an authentication that token clients must provide to the server when connecting. """ - super(JavaParameters, self).__init__( + super().__init__( address, port, auto_field, auto_close, auto_convert, eager_load, ssl_context, enable_memory_management, read_timeout, auth_token) self.auto_gc = auto_gc @@ -186,7 +186,7 @@ def __init__( :param auth_token: if provided, an authentication token that clients must provide to the server when connecting. """ - super(PythonParameters, self).__init__( + super().__init__( address, port, daemonize, daemonize_connections, eager_load, ssl_context, accept_timeout, read_timeout, propagate_java_exceptions, auth_token) @@ -216,7 +216,7 @@ def __init__( :param finalizer_deque: deque used to manage garbage collection requests. """ - super(JavaClient, self).__init__( + super().__init__( java_parameters, gateway_property=gateway_property) self.java_parameters = java_parameters @@ -233,7 +233,7 @@ def garbage_collect_object(self, target_id, enqueue=True): if enqueue: self.finalizer_deque.appendleft((self, target_id)) else: - super(JavaClient, self).garbage_collect_object(target_id) + super().garbage_collect_object(target_id) def set_thread_connection(self, connection): """Associates a ClientServerConnection with the current thread. @@ -248,7 +248,7 @@ def set_thread_connection(self, connection): def shutdown_gateway(self): try: - super(JavaClient, self).shutdown_gateway() + super().shutdown_gateway() finally: self.finalizer_deque.appendleft(SHUTDOWN_FINALIZER_WORKER) @@ -291,7 +291,7 @@ def _create_new_connection(self): def _should_retry(self, retry, connection, pne=None): # Only retry if Python was driving the communication. - parent_retry = super(JavaClient, self)._should_retry( + parent_retry = super()._should_retry( retry, connection, pne) return parent_retry and retry and connection and\ connection.initiated_from_client @@ -360,7 +360,7 @@ def __init__( :param gateway_property: used to keep gateway preferences. """ - super(PythonServer, self).__init__( + super().__init__( pool=gateway_property.pool, gateway_client=java_client, callback_server_parameters=python_parameters) @@ -450,10 +450,7 @@ def connect_to_java_server(self): def _authenticate_connection(self): if self.java_parameters.auth_token: - cmd = "{0}\n{1}\n".format( - proto.AUTH_COMMAND_NAME, - self.java_parameters.auth_token - ) + cmd = f"{proto.AUTH_COMMAND_NAME}\n{self.java_parameters.auth_token}\n" answer = self.send_command(cmd) error, _ = proto.is_error(answer) if error: @@ -498,10 +495,10 @@ def shutdown_socket(self, remote_port, local_port): logger.info( "Send shutdown request for the Java socket {0}, remote port {1}, local port {2}". format(address, remote_port, local_port)) - self.socket.sendall("z\n".encode("utf-8")) - self.socket.sendall(("%s\n" % address).encode("utf-8")) - self.socket.sendall(("%s\n" % remote_port).encode("utf-8")) - self.socket.sendall(("%s\n" % local_port).encode("utf-8")) + self.socket.sendall(b"z\n") + self.socket.sendall(f"{address}\n".encode("utf-8")) + self.socket.sendall(f"{remote_port}\n".encode("utf-8")) + self.socket.sendall(f"{local_port}\n".encode("utf-8")) logger.info("Close connection") self.close() self.is_connected = False @@ -520,18 +517,18 @@ def run(self): def send_command(self, command): # TODO At some point extract common code from wait_for_commands - logger.debug("Command to send: {0}".format(command)) + logger.debug(f"Command to send: {command}") try: self.socket.sendall(command.encode("utf-8")) except Exception as e: logger.info("Error while sending or receiving.", exc_info=True) raise Py4JNetworkError( - "Error while sending", e, proto.ERROR_ON_SEND) + "Error while sending", e, proto.ERROR_ON_SEND) from e try: while True: answer = self.stream.readline()[:-1].decode("utf-8") - logger.debug("Answer received: {0}".format(answer)) + logger.debug(f"Answer received: {answer}") # Happens when a the other end is dead. There might be an empty # answer before the socket raises an error. if answer.strip() == "": @@ -552,7 +549,7 @@ def send_command(self, command): self.socket.sendall( proto.SUCCESS_RETURN_MESSAGE.encode("utf-8")) else: - logger.error("Unknown command {0}".format(command)) + logger.error(f"Unknown command {command}") # We're sending something to prevent blocking, # but at this point, the protocol is broken. self.socket.sendall( @@ -611,7 +608,7 @@ def wait_for_commands(self): self.socket.sendall( proto.SUCCESS_RETURN_MESSAGE.encode("utf-8")) else: - logger.error("Unknown command {0}".format(command)) + logger.error(f"Unknown command {command}") # We're sending something to prevent blocking, but at this # point, the protocol is broken. self.socket.sendall( @@ -701,7 +698,7 @@ def __init__( python_parameters = PythonParameters() self.java_parameters = java_parameters self.python_parameters = python_parameters - super(ClientServer, self).__init__( + super().__init__( gateway_parameters=java_parameters, callback_server_parameters=python_parameters, python_server_entry_point=python_server_entry_point diff --git a/py4j-python/src/py4j/java_collections.py b/py4j-python/src/py4j/java_collections.py index b478309d..7f020f87 100644 --- a/py4j-python/src/py4j/java_collections.py +++ b/py4j-python/src/py4j/java_collections.py @@ -88,7 +88,7 @@ def __str__(self): def __repr__(self): items = ( - "{0}: {1}".format(repr(k), repr(v)) + f"{repr(k)}: {repr(v)}" for k, v in self.items()) return "{{{0}}}".format(", ".join(items)) @@ -190,8 +190,7 @@ def __getitem__(self, key): elif isinstance(key, int): return self.__compute_item(key) else: - raise TypeError("array indices must be integers, not {0}".format( - key.__class__.__name__)) + raise TypeError(f"array indices must be integers, not {key.__class__.__name__}") def __repl_item_from_slice(self, range, iterable): value_iter = iter(iterable) @@ -220,15 +219,14 @@ def __setitem__(self, key, value): if lenr != lenv: raise ValueError( "attempt to assign sequence of size " - "{0} to extended slice of size {1}".format(lenv, lenr)) + f"{lenv} to extended slice of size {lenr}") else: return self.__repl_item_from_slice(self_range, value) elif isinstance(key, int): return self.__set_item(key, value) else: - raise TypeError("list indices must be integers, not {0}".format( - key.__class__.__name__)) + raise TypeError(f"list indices must be integers, not {key.__class__.__name__}") def __len__(self): command = proto.ARRAY_COMMAND_NAME +\ @@ -334,15 +332,14 @@ def __setitem__(self, key, value): if lenr != lenv: raise ValueError( "attempt to assign sequence of size " - "{0} to extended slice of size {1}".format(lenv, lenr)) + f"{lenv} to extended slice of size {lenr}") else: return self.__repl_item_from_slice(self_range, value) elif isinstance(key, int): return self.__set_item(key, value) else: - raise TypeError("list indices must be integers, not {0}".format( - key.__class__.__name__)) + raise TypeError(f"list indices must be integers, not {key.__class__.__name__}") def __get_slice(self, indices): command = proto.LIST_COMMAND_NAME +\ @@ -361,8 +358,7 @@ def __getitem__(self, key): elif isinstance(key, int): return self.__compute_item(key) else: - raise TypeError("list indices must be integers, not {0}".format( - key.__class__.__name__)) + raise TypeError(f"list indices must be integers, not {key.__class__.__name__}") def __delitem__(self, key): if isinstance(key, slice): @@ -374,8 +370,7 @@ def __delitem__(self, key): elif isinstance(key, int): return self.__del_item(key) else: - raise TypeError("list indices must be integers, not {0}".format( - key.__class__.__name__)) + raise TypeError(f"list indices must be integers, not {key.__class__.__name__}") def __contains__(self, item): return self.contains(item) @@ -421,8 +416,7 @@ def insert(self, key, value): new_key = self.__compute_index(key, True) return self.add(new_key, value) else: - raise TypeError("list indices must be integers, not {0}".format( - key.__class__.__name__)) + raise TypeError(f"list indices must be integers, not {key.__class__.__name__}") def extend(self, other_list): self.addAll(other_list) @@ -472,7 +466,7 @@ def __str__(self): def __repr__(self): items = (repr(x) for x in self) - return "[{0}]".format(", ".join(items)) + return f"[{", ".join(items)}]" class SetConverter(object): diff --git a/py4j-python/src/py4j/java_gateway.py b/py4j-python/src/py4j/java_gateway.py index 0792fa22..b5ce69f7 100644 --- a/py4j-python/src/py4j/java_gateway.py +++ b/py4j-python/src/py4j/java_gateway.py @@ -211,8 +211,8 @@ def find_jar_path(): """Tries to find the path where the py4j jar is located. """ paths = [] - jar_file = "py4j{0}.jar".format(__version__) - maven_jar_file = "py4j-{0}.jar".format(__version__) + jar_file = f"py4j{__version__}.jar" + maven_jar_file = f"py4j-{__version__}.jar" paths.append(jar_file) # ant paths.append(os.path.join(os.path.dirname( @@ -319,7 +319,7 @@ def launch_gateway(port=0, jarpath="", classpath="", javaopts=[], # Fail if the jar does not exist. if not os.path.exists(jarpath): - raise Py4JError("Could not find py4j jar at {0}".format(jarpath)) + raise Py4JError(f"Could not find py4j jar at {jarpath}") # Launch the server in a subprocess. classpath = os.pathsep.join((jarpath, classpath)) @@ -330,7 +330,7 @@ def launch_gateway(port=0, jarpath="", classpath="", javaopts=[], if enable_auth: command.append("--enable-auth") command.append(str(port)) - logger.debug("Launching gateway with command {0}".format(command)) + logger.debug(f"Launching gateway with command {command}") # stderr redirection close_stderr = False @@ -414,8 +414,7 @@ def get_field(java_object, field_name): if answer == proto.NO_MEMBER_COMMAND or has_error: message = compute_exception_message( - "no field {0} in object {1}".format( - field_name, java_object._target_id), error_message) + f"no field {field_name} in object {java_object._target_id}", error_message) raise Py4JError(message) else: return get_return_value( @@ -447,8 +446,7 @@ def set_field(java_object, field_name, value): if answer == proto.NO_MEMBER_COMMAND or has_error: message = compute_exception_message( - "no field {0} in object {1}".format( - field_name, java_object._target_id), error_message) + f"no field {field_name} in object {java_object._target_id}", error_message) raise Py4JError(message) return get_return_value( answer, java_object._gateway_client, java_object._target_id, @@ -638,8 +636,8 @@ def do_client_auth(command, input_stream, sock, auth_token): """ try: if command != proto.AUTH_COMMAND_NAME: - raise Py4JAuthenticationError("Expected {}, received {}.".format( - proto.AUTH_COMMAND_NAME, command)) + raise Py4JAuthenticationError( + f"Expected {proto.AUTH_COMMAND_NAME}, received {command}.") client_token = input_stream.readline()[:-1].decode("utf-8") # Remove the END marker @@ -717,8 +715,7 @@ def _garbage_collect_proxy(pool, proxy_id): success = True except KeyError: logger.warning( - "Tried to garbage collect non existing python proxy {0}" - .format(proxy_id)) + f"Tried to garbage collect non existing python proxy {proxy_id}") return success @@ -727,7 +724,7 @@ class OutputConsumer(Thread): """ def __init__(self, redirect, stream, *args, **kwargs): - super(OutputConsumer, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.redirect = redirect self.stream = stream @@ -759,7 +756,7 @@ class ProcessConsumer(Thread): """ def __init__(self, proc, closable_list, *args, **kwargs): - super(ProcessConsumer, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.proc = proc if closable_list: # We don't care if it contains queues or deques, quiet_close will @@ -1115,11 +1112,11 @@ def send_command(self, command, retry=True, binary=False): (next_conn.socket.getsockname() == local_addr or \ next_conn.socket.getpeername() == remote_addr) if socket_match: - logger.info("Shutting down matched socket {0}".format(next_conn.socket)) + logger.info(f"Shutting down matched socket {next_conn.socket}") # We send local port as remote and remote port as local for JVM part # of the connection. next_conn.shutdown_socket(local_addr[1], remote_addr[1]) - logger.info("Finished shutdown of socket {0}".format(next_conn.socket)) + logger.info(f"Finished shutdown of socket {next_conn.socket}") else: remaining_sockets.append(next_conn) except IndexError: @@ -1128,7 +1125,7 @@ def send_command(self, command, retry=True, binary=False): for conn in remaining_sockets: self.deque.append(conn) - logger.info("Shutting down the current connection {0}".format(connection)) + logger.info(f"Shutting down the current connection {connection}") # The ports are reversed for JVM side of the connection. connection.shutdown_socket(local_addr[1], remote_addr[1]) @@ -1207,16 +1204,13 @@ def start(self): raise except Exception as e: msg = "An error occurred while trying to connect to the Java "\ - "server ({0}:{1})".format(self.address, self.port) + f"server ({self.address}:{self.port})" logger.exception(msg) - raise Py4JNetworkError(msg, e) + raise Py4JNetworkError(msg, e) from e def _authenticate_connection(self): if self.gateway_parameters.auth_token: - cmd = "{0}\n{1}\n".format( - proto.AUTH_COMMAND_NAME, - self.gateway_parameters.auth_token - ) + cmd = f"{proto.AUTH_COMMAND_NAME}\n{self.gateway_parameters.auth_token}\n" answer = self.send_command(cmd) error, _ = proto.is_error(answer) if error: @@ -1271,7 +1265,7 @@ def send_command(self, command): :rtype: the `string` answer received from the JVM (The answer follows the Py4J protocol). """ - logger.debug("Command to send: {0}".format(command)) + logger.debug(f"Command to send: {command}") try: # Write will only fail if remote is closed for large payloads or # if it sent a RST packet (SO_LINGER) @@ -1279,7 +1273,7 @@ def send_command(self, command): except Exception as e: logger.info("Error while sending.", exc_info=True) raise Py4JNetworkError( - "Error while sending", e, proto.ERROR_ON_SEND) + "Error while sending", e, proto.ERROR_ON_SEND) from e try: # Stream is opened in binary mode (socket.makefile("rb")), @@ -1287,7 +1281,7 @@ def send_command(self, command): # than dispatch through smart_decode's isinstance check. # Every JavaGateway call hits this — the saving compounds. answer = self.stream.readline()[:-1].decode("utf-8") - logger.debug("Answer received: {0}".format(answer)) + logger.debug(f"Answer received: {answer}") if answer.startswith(proto.RETURN_MESSAGE): answer = answer[1:] # Happens when a the other end is dead. There might be an empty @@ -1606,7 +1600,7 @@ def _java_lang_class(self): answer, self._gateway_client, self._fqn, "_java_lang_class") else: raise Py4JError( - "{0} does not exist in the JVM".format(self._fqn)) + f"{self._fqn} does not exist in the JVM") def __getattr__(self, name): if is_magic_member(name): @@ -1633,7 +1627,7 @@ def __getattr__(self, name): answer, self._gateway_client, self._fqn, name) else: raise Py4JError( - "{0}.{1} does not exist in the JVM".format(self._fqn, name)) + f"{self._fqn}.{name} does not exist in the JVM") def _get_args(self, args): temp_args = [] @@ -1747,7 +1741,7 @@ def __getattr__(self, name): return JavaClass( answer[proto.CLASS_FQN_START:], self._gateway_client) else: - raise Py4JError("{0} does not exist in the JVM".format(new_fqn)) + raise Py4JError(f"{new_fqn} does not exist in the JVM") class JVMView(object): @@ -1807,7 +1801,7 @@ def __getattr__(self, name): else: _, error_message = get_error_message(answer) message = compute_exception_message( - "{0} does not exist in the JVM".format(name), error_message) + f"{name} does not exist in the JVM", error_message) raise Py4JError(message) @@ -2321,9 +2315,9 @@ def start(self): self._listening_port = info[1] except Exception as e: msg = "An error occurred while trying to start the callback "\ - "server ({0}:{1})".format(self.address, self.port) + f"server ({self.address}:{self.port})" logger.exception(msg) - raise Py4JNetworkError(msg, e) + raise Py4JNetworkError(msg, e) from e # Maybe thread needs to be cleanup up? self.thread = Thread(target=self.run) @@ -2460,7 +2454,7 @@ class CallbackConnection(Thread): def __init__( self, pool, input, socket_instance, gateway_client, callback_server_parameters, callback_server): - super(CallbackConnection, self).__init__() + super().__init__() self.pool = pool self.input = input self.socket = socket_instance @@ -2506,7 +2500,7 @@ def run(self): self.socket.sendall( proto.SUCCESS_RETURN_MESSAGE.encode("utf-8")) else: - logger.error("Unknown command {0}".format(command)) + logger.error(f"Unknown command {command}") # We're sending something to prevent blokincg, but at this # point, the protocol is broken. self.socket.sendall( diff --git a/py4j-python/src/py4j/protocol.py b/py4j-python/src/py4j/protocol.py index bb14d36a..c82a0151 100644 --- a/py4j-python/src/py4j/protocol.py +++ b/py4j-python/src/py4j/protocol.py @@ -395,8 +395,7 @@ def compute_exception_message(default_message, extra_message=None): """ message = default_message if extra_message: - message = "{0} -- {1}".format( - default_message, extra_message) + message = f"{default_message} -- {extra_message}" return message @@ -457,21 +456,21 @@ class Py4JError(Exception): """Exception raised when a problem occurs with Py4J.""" def __init__(self, args=None, cause=None): - super(Py4JError, self).__init__(args) + super().__init__(args) self.cause = cause class Py4JAuthenticationError(Py4JError): """Exception raised when Py4J cannot authenticate a connection.""" def __init__(self, args=None, cause=None): - super(Py4JAuthenticationError, self).__init__(args) + super().__init__(args) self.cause = cause class Py4JNetworkError(Py4JError): """Exception raised when a network error occurs with Py4J.""" def __init__(self, args=None, cause=None, when=None): - super(Py4JNetworkError, self).__init__(args) + super().__init__(args) self.cause = cause self.when = when @@ -498,4 +497,4 @@ def __str__(self): gateway_client = self.java_exception._gateway_client answer = gateway_client.send_command(self.exception_cmd) return_value = get_return_value(answer, gateway_client, None, None) - return "{0}: {1}".format(self.errmsg, return_value) + return f"{self.errmsg}: {return_value}" From 309900741209a33262a8196c9376a8af972c3f0b Mon Sep 17 00:00:00 2001 From: Ruslan Dautkhanov Date: Thu, 21 May 2026 11:57:57 -0600 Subject: [PATCH 3/4] fix: avoid nested quotes in f-string for Python 3.9-3.11 compat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit's mechanical .format() -> f-string converter emitted this on java_collections.py:469: return f"[{", ".join(items)}]" PEP 701 (nested quotes in f-strings) only landed in Python 3.12, so this is a SyntaxError on 3.9 / 3.10 / 3.11 — the entire module fails to import, and any test that uses java_collections crashes at collection time. CI surfaced this on Python 3.11, Java 8, ubuntu- latest first; other 3.9-3.11 cells were on the same trajectory. Refactor the call site to extract the ", ".join() into a local binding outside the f-string. Equivalent semantics, parses cleanly on every supported Python. The original .format()-to-f-string converter has been mentally patched: if a .format() arg contains a string literal, the resulting f-string would produce nested quotes — skip such sites or extract to a local. (No other sites in this PR have that pattern; this was the only instance.) Co-authored-by: Isaac --- py4j-python/src/py4j/java_collections.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/py4j-python/src/py4j/java_collections.py b/py4j-python/src/py4j/java_collections.py index 7f020f87..fa1b70c4 100644 --- a/py4j-python/src/py4j/java_collections.py +++ b/py4j-python/src/py4j/java_collections.py @@ -466,7 +466,8 @@ def __str__(self): def __repr__(self): items = (repr(x) for x in self) - return f"[{", ".join(items)}]" + inside = ", ".join(items) + return f"[{inside}]" class SetConverter(object): From 590b2e47f94ff217cb65d79e52eb856ccb697913 Mon Sep 17 00:00:00 2001 From: Ruslan Dautkhanov Date: Thu, 21 May 2026 11:52:58 -0600 Subject: [PATCH 4/4] types: add Python type hints to public API + low-risk internals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds annotations via PEP 563 (from __future__ import annotations) so all annotations are strings at runtime — zero overhead, zero introspection surprises for PySpark or any other consumer. Annotated: * java_gateway.py public API: GatewayParameters / CallbackServerParameters / JavaGateway __init__ signatures; launch_gateway; java_import; get_field / set_field; is_instance_of; exception class constructors (Py4JError, Py4JNetworkError, Py4JAuthenticationError, Py4JJavaError). * protocol.py: encode/decode function signatures (escape_new_line, unescape_new_line, smart_decode, encode_float, encode_bytearray, decode_bytearray, is_python_proxy, get_command_part, get_return_value) plus is_error, is_fatal_error. * signals.py: Signal.connect / disconnect / send. * finalizer.py: ThreadSafeFinalizer.add_finalizer / remove_finalizer / clear_finalizers. Deliberately NOT annotated: * JavaObject / JavaMember / JavaClass / JavaPackage / JVMView — attribute access resolves dynamically via __getattr__ + Java reflection; annotating would either lie about specific types or be pure Any. Honest stance: leave dynamic dispatch unannotated. * java_collections.py (JavaList/JavaMap/JavaSet) — same dynamic dispatch pattern. * GatewayClient / CallbackServer internal methods that touch JavaObject — same reason. mypy is not gated in CI for this PR (deferred per spec); annotations are added for IDE / static-analysis benefit. A future discussion on style / lint tooling (likely covering ruff + black + mypy together) will revisit CI gating. Co-authored-by: Isaac --- py4j-python/src/py4j/finalizer.py | 9 ++- py4j-python/src/py4j/java_gateway.py | 82 +++++++++++++++++++--------- py4j-python/src/py4j/protocol.py | 36 +++++++----- py4j-python/src/py4j/signals.py | 9 ++- 4 files changed, 89 insertions(+), 47 deletions(-) diff --git a/py4j-python/src/py4j/finalizer.py b/py4j-python/src/py4j/finalizer.py index c131b44b..191d6df8 100644 --- a/py4j-python/src/py4j/finalizer.py +++ b/py4j-python/src/py4j/finalizer.py @@ -6,7 +6,10 @@ :author: Barthelemy Dagenais """ +from __future__ import annotations + from threading import RLock +from typing import Any class ThreadSafeFinalizer(object): @@ -28,7 +31,7 @@ class ThreadSafeFinalizer(object): lock = RLock() @classmethod - def add_finalizer(cls, id, weak_ref): + def add_finalizer(cls, id: Any, weak_ref: Any) -> None: """Registers a finalizer with an id. :param id: The id of the object referenced by the weak reference. @@ -38,7 +41,7 @@ def add_finalizer(cls, id, weak_ref): cls.finalizers[id] = weak_ref @classmethod - def remove_finalizer(cls, id): + def remove_finalizer(cls, id: Any) -> None: """Removes a finalizer associated with this id. :param id: The id of the object for which the finalizer will be @@ -48,7 +51,7 @@ def remove_finalizer(cls, id): cls.finalizers.pop(id, None) @classmethod - def clear_finalizers(cls, clear_all=False): + def clear_finalizers(cls, clear_all: bool = False) -> None: """Removes all registered finalizers. :param clear_all: If `True`, all finalizers are deleted. Otherwise, diff --git a/py4j-python/src/py4j/java_gateway.py b/py4j-python/src/py4j/java_gateway.py index b5ce69f7..5ab26b3d 100644 --- a/py4j-python/src/py4j/java_gateway.py +++ b/py4j-python/src/py4j/java_gateway.py @@ -9,6 +9,8 @@ :author: Barthelemy Dagenais """ +from __future__ import annotations + from collections import deque import logging import os @@ -16,12 +18,14 @@ from queue import Queue import select import socket +import ssl import struct from subprocess import Popen, PIPE import subprocess import sys import traceback from threading import Thread, RLock +from typing import Any import weakref from py4j.compat import hasattr2 @@ -190,7 +194,7 @@ def deprecated(name, last_version, use_instead="", level=logging.DEBUG, raise DeprecationWarning(msg) -def java_import(jvm_view, import_str): +def java_import(jvm_view: Any, import_str: str) -> None: """Imports the package or class specified by `import_str` in the jvm view namespace. @@ -242,12 +246,21 @@ def find_jar_path(): return "" -def launch_gateway(port=0, jarpath="", classpath="", javaopts=[], - die_on_exit=False, redirect_stdout=None, - redirect_stderr=None, daemonize_redirect=True, - java_path="java", create_new_process_group=False, - enable_auth=False, cwd=None, return_proc=False, - use_shell=False): +def launch_gateway( + port: int = 0, + jarpath: str = "", + classpath: str = "", + javaopts: list[str] = [], + die_on_exit: bool = False, + redirect_stdout: Any = None, + redirect_stderr: Any = None, + daemonize_redirect: bool = True, + java_path: str = "java", + create_new_process_group: bool = False, + enable_auth: bool = False, + cwd: str | None = None, + return_proc: bool = False, + use_shell: bool = False) -> int | tuple[int, Popen] | tuple[int, str] | tuple[int, str, Popen]: """Launch a `Gateway` in a new Java process. The redirect parameters accept file-like objects, Queue, or deque. When @@ -397,7 +410,7 @@ def launch_gateway(port=0, jarpath="", classpath="", javaopts=[], return output -def get_field(java_object, field_name): +def get_field(java_object: Any, field_name: str) -> Any: """Retrieves the field named `field_name` from the `java_object`. This function is useful when `auto_field=false` in a gateway or @@ -422,7 +435,7 @@ def get_field(java_object, field_name): field_name) -def set_field(java_object, field_name, value): +def set_field(java_object: Any, field_name: str, value: Any) -> None: """Sets the field named `field_name` of `java_object` to `value`. This function is the only way to set a field because the assignment @@ -468,7 +481,7 @@ def get_method(java_object, method_name): java_object._gateway_client) -def is_instance_of(gateway, java_object, java_class): +def is_instance_of(gateway: JavaGateway, java_object: Any, java_class: Any) -> bool: """Indicates whether a java object is an instance of the provided java_class. @@ -779,10 +792,17 @@ class GatewayParameters(object): """ def __init__( - self, address=DEFAULT_ADDRESS, port=DEFAULT_PORT, auto_field=False, - auto_close=True, auto_convert=False, eager_load=False, - ssl_context=None, enable_memory_management=True, - read_timeout=None, auth_token=None): + self, + address: str = DEFAULT_ADDRESS, + port: int = DEFAULT_PORT, + auto_field: bool = False, + auto_close: bool = True, + auto_convert: bool = False, + eager_load: bool = False, + ssl_context: ssl.SSLContext | None = None, + enable_memory_management: bool = True, + read_timeout: float | None = None, + auth_token: str | None = None) -> None: """ :param address: the address to which the client will request a connection. If you're assing a `SSLContext` with @@ -842,12 +862,17 @@ class CallbackServerParameters(object): """ def __init__( - self, address=DEFAULT_ADDRESS, port=DEFAULT_PYTHON_PROXY_PORT, - daemonize=False, daemonize_connections=False, eager_load=True, - ssl_context=None, - accept_timeout=DEFAULT_ACCEPT_TIMEOUT_PLACEHOLDER, - read_timeout=None, propagate_java_exceptions=False, - auth_token=None): + self, + address: str = DEFAULT_ADDRESS, + port: int = DEFAULT_PYTHON_PROXY_PORT, + daemonize: bool = False, + daemonize_connections: bool = False, + eager_load: bool = True, + ssl_context: ssl.SSLContext | None = None, + accept_timeout: float | str = DEFAULT_ACCEPT_TIMEOUT_PLACEHOLDER, + read_timeout: float | None = None, + propagate_java_exceptions: bool = False, + auth_token: str | None = None) -> None: """ :param address: the address to which the client will request a connection @@ -1843,12 +1868,17 @@ class JavaGateway(object): """ def __init__( - self, gateway_client=None, auto_field=False, - python_proxy_port=DEFAULT_PYTHON_PROXY_PORT, - start_callback_server=False, auto_convert=False, eager_load=False, - gateway_parameters=None, callback_server_parameters=None, - python_server_entry_point=None, - java_process=None): + self, + gateway_client: GatewayClient | None = None, + auto_field: bool = False, + python_proxy_port: int = DEFAULT_PYTHON_PROXY_PORT, + start_callback_server: bool = False, + auto_convert: bool = False, + eager_load: bool = False, + gateway_parameters: GatewayParameters | None = None, + callback_server_parameters: CallbackServerParameters | None = None, + python_server_entry_point: Any = None, + java_process: Popen | None = None) -> None: """ :param gateway_parameters: An instance of `GatewayParameters` used to configure the various options of the gateway. diff --git a/py4j-python/src/py4j/protocol.py b/py4j-python/src/py4j/protocol.py index c82a0151..6a55ee81 100644 --- a/py4j-python/src/py4j/protocol.py +++ b/py4j-python/src/py4j/protocol.py @@ -16,9 +16,15 @@ :author: Barthelemy Dagenais """ +from __future__ import annotations + from base64 import standard_b64encode, standard_b64decode from decimal import Decimal +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from py4j.java_gateway import PythonProxyPool JAVA_MAX_INT = 2147483647 @@ -168,7 +174,7 @@ EMPTY_RESPONSE = "empty_response" -def escape_new_line(original): +def escape_new_line(original: str | bytes) -> str: """Replaces new line characters by a backslash followed by a n. Backslashes are also escaped by another backslash. @@ -197,7 +203,7 @@ def escape_new_line(original): return original -def unescape_new_line(escaped): +def unescape_new_line(escaped: str) -> str: """Replaces escaped characters by unescaped characters. For example, double backslashes are replaced by a single backslash. @@ -218,7 +224,7 @@ def unescape_new_line(escaped): return escaped -def smart_decode(s): +def smart_decode(s: bytes | str) -> str: if isinstance(s, str): return s elif isinstance(s, bytes): @@ -227,7 +233,7 @@ def smart_decode(s): return str(s) -def encode_float(float_value): +def encode_float(float_value: float) -> str: # str(float) on Python 3 already returns the same shortest- # roundtrip repr that smart_decode(repr(...)) was producing on # py2; smart_decode here was a no-op dispatcher. @@ -241,7 +247,7 @@ def encode_float(float_value): return float_str -def encode_bytearray(barray): +def encode_bytearray(barray: bytes | bytearray) -> str: if isinstance(barray, bytes): return str(standard_b64encode(barray), encoding="ascii") else: @@ -249,7 +255,7 @@ def encode_bytearray(barray): return str(standard_b64encode(newbytestr), encoding="ascii") -def decode_bytearray(encoded): +def decode_bytearray(encoded: str) -> bytes: # Per @PaperTsar's analysis in issue #570: the prior # implementation built a Python list of ints (one PyObject per # byte) then reconstructed bytes from that list — pure overhead @@ -260,7 +266,7 @@ def decode_bytearray(encoded): return bytes(standard_b64decode(encoded.encode("ascii"))) -def is_python_proxy(parameter): +def is_python_proxy(parameter: object) -> bool: """Determines whether parameter is a Python Proxy, i.e., it has a Java internal class with an `implements` member. @@ -275,7 +281,7 @@ def is_python_proxy(parameter): return is_proxy -def get_command_part(parameter, python_proxy_pool=None): +def get_command_part(parameter: object, python_proxy_pool: PythonProxyPool | None = None) -> str: """Converts a Python object into a string representation respecting the Py4J protocol. @@ -317,7 +323,7 @@ def get_command_part(parameter, python_proxy_pool=None): return command_part -def get_return_value(answer, gateway_client, target_id=None, name=None): +def get_return_value(answer: str, gateway_client: Any, target_id: str | None = None, name: str | None = None) -> Any: """Converts an answer received from the Java gateway into a Python object. For example, string representation of integers are converted to Python @@ -399,14 +405,14 @@ def compute_exception_message(default_message, extra_message=None): return message -def is_error(answer): +def is_error(answer: str) -> tuple[bool, None]: if len(answer) == 0 or answer[0] != SUCCESS: return (True, None) else: return (False, None) -def is_fatal_error(answer): +def is_fatal_error(answer: str) -> bool: return answer and len(answer) > 0 and answer[0] == FATAL_ERROR @@ -455,21 +461,21 @@ def register_input_converter(converter, prepend=False): class Py4JError(Exception): """Exception raised when a problem occurs with Py4J.""" - def __init__(self, args=None, cause=None): + def __init__(self, args: str | None = None, cause: BaseException | None = None) -> None: super().__init__(args) self.cause = cause class Py4JAuthenticationError(Py4JError): """Exception raised when Py4J cannot authenticate a connection.""" - def __init__(self, args=None, cause=None): + def __init__(self, args: str | None = None, cause: BaseException | None = None) -> None: super().__init__(args) self.cause = cause class Py4JNetworkError(Py4JError): """Exception raised when a network error occurs with Py4J.""" - def __init__(self, args=None, cause=None, when=None): + def __init__(self, args: str | None = None, cause: BaseException | None = None, when: str | None = None) -> None: super().__init__(args) self.cause = cause self.when = when @@ -486,7 +492,7 @@ class Py4JJavaError(Py4JError): """ - def __init__(self, msg, java_exception): + def __init__(self, msg: str, java_exception: Any) -> None: self.args = (msg, java_exception) self.errmsg = msg self.java_exception = java_exception diff --git a/py4j-python/src/py4j/signals.py b/py4j-python/src/py4j/signals.py index 1627a8d1..59fed81a 100644 --- a/py4j-python/src/py4j/signals.py +++ b/py4j-python/src/py4j/signals.py @@ -3,8 +3,11 @@ The signals pattern is very similar to the listener/observer pattern. """ +from __future__ import annotations + from inspect import ismethod from threading import Lock +from typing import Any, Callable def make_id(func): @@ -33,7 +36,7 @@ def __init__(self): # number of receivers to be very small. self.receivers = [] - def connect(self, receiver, sender=None, unique_id=None): + def connect(self, receiver: Callable[..., Any], sender: object = None, unique_id: Any = None) -> None: """Registers a receiver for this signal. The receiver must be a callable (e.g., function or instance method) @@ -58,7 +61,7 @@ def connect(self, receiver, sender=None, unique_id=None): else: self.receivers.append((full_id, receiver)) - def disconnect(self, receiver, sender=None, unique_id=None): + def disconnect(self, receiver: Callable[..., Any], sender: object = None, unique_id: Any = None) -> bool: """Unregisters a receiver for this signal. :param receiver: The callable that was registered to receive the @@ -82,7 +85,7 @@ def disconnect(self, receiver, sender=None, unique_id=None): return disconnected - def send(self, sender, **params): + def send(self, sender: object, **params: Any) -> list[tuple[Callable[..., Any], Any]]: """Sends the signal to all connected receivers. If a receiver raises an error, the error is propagated back and