feature: add lean ui terminal UI with live backtest results

README.md

@@ -143,6 +143,7 @@ The following CLI configurations are available. Use the [`lean config list`](#le
 - [`lean project-delete`](#lean-project-delete)
 - [`lean report`](#lean-report)
 - [`lean research`](#lean-research)
+- [`lean ui`](#lean-ui)
 - [`lean whoami`](#lean-whoami)
 
 ### `lean backtest`
@@ -2293,6 +2294,25 @@ Options:
 
 _See code: [lean/commands/research.py](lean/commands/research.py)_
 
+### `lean ui`
+
+Browse projects and run commands on them in an interactive terminal UI.
+
+```
+Usage: lean ui [OPTIONS]
+
+  Browse projects and run commands on them in an interactive terminal UI.
+
+  Use the arrow keys or j/k to move, / to filter, enter to pick an action and q to quit.
+
+Options:
+  --lean-config FILE  The Lean configuration file that should be used (defaults to the nearest lean.json)
+  --verbose           Enable debug logging
+  --help              Show this message and exit.
+```
+
+_See code: [lean/commands/ui.py](lean/commands/ui.py)_
+
 ### `lean whoami`
 
 Display who is logged in.

lean/__main__.py

@@ -0,0 +1,18 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Allows running the CLI with `python -m lean`, which is how `lean ui` launches commands as child processes
+
+from lean.main import main
+
+main(prog_name="lean")

lean/click.py

@@ -41,7 +41,7 @@ def get_whoami_message() -> str:
     api_client = container.api_client
     cli_config_manager = container.cli_config_manager
 
-    if cli_config_manager.user_id.get_value() is not None and cli_config_manager.api_token.get_value() is not None:
+    if cli_config_manager.has_credentials():
         try:
             organizations = api_client.organizations.get_all()
             logged_in = True

lean/commands/__init__.py

@@ -32,6 +32,7 @@
 from lean.commands.research import research
 from lean.commands.whoami import whoami
 from lean.commands.gui import gui
+from lean.commands.ui import ui
 from lean.commands.object_store import object_store
 from lean.commands.private_cloud import private_cloud
 
@@ -55,5 +56,6 @@
 lean.add_command(build)
 lean.add_command(logs)
 lean.add_command(gui)
+lean.add_command(ui)
 lean.add_command(object_store)
 lean.add_command(private_cloud)

lean/commands/ui.py

@@ -0,0 +1,37 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from click import command
+
+from lean.click import LeanCommand
+from lean.container import container
+
+
+# requires_docker makes the CLI elevate with sudo on Linux before the UI starts, the commands it runs have no terminal
+@command(cls=LeanCommand, requires_lean_config=True, requires_docker=True)
+def ui() -> None:
+    """Browse projects and run commands on them in an interactive terminal UI.
+
+    \b
+    Use the arrow keys or j/k to move, / to filter, enter to pick an action and q to quit.
+    """
+    # Textual is imported here so it does not slow down the startup of other commands
+    from lean.ui.app import LeanApp
+
+    lean_config_manager = container.lean_config_manager
+    try:
+        data_dir = lean_config_manager.get_data_directory()
+    except Exception:
+        data_dir = None
+
+    LeanApp(lean_config_manager.get_cli_root_directory(), data_dir).run()

lean/components/api/backtest_client.py

@@ -11,13 +11,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import Any, Dict, List, Optional
+
 from lean.components.api.api_client import *
 from lean.models.api import QCBacktest
 
 
 class BacktestClient:
     """The BacktestClient class contains methods to interact with backtests/* API endpoints."""
 
+    # How often and how long to wait for the API to prepare a backtest's chart, orders or log
+    _LOADING_ATTEMPTS = 30
+    _LOADING_DELAY_SECONDS = 1
+
     def __init__(self, api_client: 'APIClient') -> None:
         """Creates a new BacktestClient instance.
 
@@ -73,3 +79,69 @@ def delete(self, project_id: int, backtest_id: str) -> None:
             "projectId": project_id,
             "backtestId": backtest_id
         })
+
+    def list(self, project_id: int) -> List[Dict[str, Any]]:
+        """Returns the backtests of a project without their statistics.
+
+        :param project_id: the id of the project
+        :return: the backtests, each holding at least its backtestId, name and created time
+        """
+        data = self._api.post("backtests/list", {"projectId": project_id, "includeStatistics": False})
+        return data.get("backtests") or []
+
+    def read_chart(self, project_id: int, backtest_id: str, name: str, start: int, end: int,
+                   count: int) -> Optional[Dict[str, Any]]:
+        """Returns a chart of a backtest, sampled down to the given number of points.
+
+        :param project_id: the id of the project the backtest belongs to
+        :param backtest_id: the id of the backtest
+        :param name: the name of the chart
+        :param start: the start of the range to read, as a unix timestamp
+        :param end: the end of the range to read, as a unix timestamp
+        :param count: the number of points to sample the chart down to
+        :return: the chart, or None if it is not available yet
+        """
+        data = self._post_when_ready("backtests/chart/read", project_id, backtest_id,
+                                     {"name": name, "count": count, "start": start, "end": end})
+        return data.get("chart") or None
+
+    def read_orders(self, project_id: int, backtest_id: str, start: int, end: int) -> Dict[str, Any]:
+        """Returns a page of the orders of a backtest, the API serves at most 100 orders per request.
+
+        :param project_id: the id of the project the backtest belongs to
+        :param backtest_id: the id of the backtest
+        :param start: the index of the first order to read
+        :param end: the index after the last order to read
+        :return: the page, "orders" holding the orders and "length" the total number of orders
+        """
+        return self._post_when_ready("backtests/orders/read", project_id, backtest_id, {"start": start, "end": end})
+
+    def read_log(self, project_id: int, backtest_id: str, start: int, end: int) -> Dict[str, Any]:
+        """Returns a page of the log of a backtest, the API serves at most 200 lines per request.
+
+        :param project_id: the id of the project the backtest belongs to
+        :param backtest_id: the id of the backtest
+        :param start: the index of the first line to read
+        :param end: the index after the last line to read
+        :return: the page, "logs" holding the lines and "length" the total number of lines
+        """
+        return self._post_when_ready("backtests/read/log", project_id, backtest_id,
+                                     {"start": start, "end": end, "query": " "})
+
+    def _post_when_ready(self, endpoint: str, project_id: int, backtest_id: str,
+                         payload: Optional[Dict[str, Any]] = None) -> Any:
+        """Makes a request about a backtest, retrying while the API is still preparing the data.
+
+        The chart, orders and log endpoints prepare their data the first time it is asked for and answer
+        {"status": "loading", "progress": ...} until it is ready.
+        """
+        from time import sleep
+
+        data = {"projectId": project_id, "backtestId": backtest_id, **(payload or {})}
+        for attempt in range(self._LOADING_ATTEMPTS):
+            response = self._api.post(endpoint, data)
+            if response.get("status") != "loading":
+                return response
+            if attempt < self._LOADING_ATTEMPTS - 1:
+                sleep(self._LOADING_DELAY_SECONDS)
+        raise RuntimeError(f"QuantConnect is still preparing the data of {endpoint}, try again later")

lean/components/config/cli_config_manager.py

@@ -74,6 +74,13 @@ def __init__(self, general_storage: Storage, credentials_storage: Storage) -> No
             self.database_update_frequency
         ]
 
+    def has_credentials(self) -> bool:
+        """Returns whether a user id and API token are stored, without checking them against the API.
+
+        :return: True if both credentials are set, False if not
+        """
+        return self.user_id.get_value() is not None and self.api_token.get_value() is not None
+
     def get_option_by_key(self, key: str) -> Option:
         """Returns the option matching the given key.
 

lean/components/docker/docker_manager.py

@@ -103,6 +103,7 @@ def run_image(self, image: DockerImage, **kwargs) -> bool:
         from types import FrameType
         from docker.errors import APIError
         from docker.types import Mount
+        from lean.constants import IS_UI_COMMAND
 
         if not self.image_installed(image):
             self.pull_image(image)
@@ -143,6 +144,8 @@ def run_image(self, image: DockerImage, **kwargs) -> bool:
 
         detach = kwargs.pop("detach", False)
         is_tty = stdout.isatty()
+        # Without a terminal there is nobody to interrupt twice, except `lean ui` which stops its commands that way
+        graceful_stop = is_tty or IS_UI_COMMAND
 
         kwargs["detach"] = True
         kwargs["hostname"] = kwargs["hostname"] if "hostname" in kwargs else node()
@@ -224,7 +227,7 @@ def kill_container(force: bool) -> None:
         # Kill the container on Ctrl+C
         def signal_handler(sig: Signals, frame: FrameType) -> None:
             nonlocal force_kill_next
-            if not is_tty or force_kill_next or kwargs["stop_signal"] == "SIGKILL":
+            if not graceful_stop or force_kill_next or kwargs["stop_signal"] == "SIGKILL":
                 force_kill_current = True
             else:
                 self._logger.info("Waiting 1 minute for LEAN to exit gracefully, press Ctrl+C again to force stop")

lean/constants.py

@@ -122,3 +122,7 @@
 
 # Local backtests, optimizations and live deployments write their output to a directory named after their start time
 OUTPUT_DIRECTORY_NAME_FORMAT = "%Y-%m-%d_%H-%M-%S"
+
+# Set by `lean ui` in the commands it runs, which it stops with Ctrl+Break on Windows
+UI_COMMAND_ENV_VAR = "LEAN_UI_COMMAND"
+IS_UI_COMMAND = environ.get(UI_COMMAND_ENV_VAR) == "1"

lean/main.py

@@ -88,10 +88,28 @@ def _ensure_win32_available() -> None:
 from lean.container import container
 
 
-def main() -> None:
-    """This function is the entrypoint when running a Lean command in a terminal."""
+def _handle_ctrl_break_as_ctrl_c() -> None:
+    """On Windows `lean ui` stops a command with Ctrl+Break, which must run the command's Ctrl+C cleanup."""
+    from lean.constants import IS_UI_COMMAND
+    if not IS_UI_COMMAND:
+        return
+    from signal import SIGINT, raise_signal, signal
+    try:
+        from signal import SIGBREAK
+    except ImportError:
+        return
+    signal(SIGBREAK, lambda *_: raise_signal(SIGINT))
+
+
+def main(prog_name: str = None) -> None:
+    """This function is the entrypoint when running a Lean command in a terminal.
+
+    :param prog_name: the program name shown in help and error messages, detected from the command line if not set
+    """
+    _handle_ctrl_break_as_ctrl_c()
+
     try:
-        lean.main(standalone_mode=False)
+        lean.main(standalone_mode=False, prog_name=prog_name)
 
         temp_manager = container.temp_manager
         if temp_manager.delete_temporary_directories_when_done:

lean/models/api.py

@@ -185,6 +185,10 @@ class QCBacktest(WrappedBaseModel):
     runtimeStatistics: Optional[Dict[str, str]] = None
     statistics: Optional[Union[Dict[str, str], List[Any]]] = None
     totalPerformance: Optional[Any] = None
+    backtestStart: Optional[Any] = None
+    backtestEnd: Optional[Any] = None
+    charts: Optional[Union[Dict[str, Any], List[Any]]] = None
+    parameterSet: Optional[Any] = None
 
     def is_complete(self) -> bool:
         """Returns whether the backtest has completed in the cloud.

lean/ui/__init__.py

@@ -0,0 +1,12 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.