diff --git a/.github/actions/unity-gallery-prebuilt/action.yml b/.github/actions/unity-gallery-prebuilt/action.yml new file mode 100644 index 00000000000..c17a66518e7 --- /dev/null +++ b/.github/actions/unity-gallery-prebuilt/action.yml @@ -0,0 +1,53 @@ +name: Unpack the Unity gallery prebuilt +description: >- + Downloads the compiled Unity gallery that unity-gallery-prebuilt.yml uploaded + in this run and puts it where a build of scripts/hellocodenameone reads it, + so the job needs neither the .NET SDK nor the Unity modules. + +# scripts/hellocodenameone compiles a Unity project +# (scripts/unity-compat-samples/gallery). Compiling it takes the .NET SDK and +# the two Maven modules behind -Dunity-compat; one Linux job per workflow run +# does that (.github/workflows/unity-gallery-prebuilt.yml) and every job that +# builds the application uses this action instead. +# +# Where it goes in a job: +# - AFTER the checkout (the action is read from the workspace) and after +# anything that restores or installs the framework into the local Maven +# repository: a cache or a bundle restored later could put another +# codenameone-unity-compat jar back over the one unpacked here, and the +# compiled output is good only with the exact jars of the archive. +# - BEFORE the first Maven command that builds the application. +# - The job must `needs:` the job that calls unity-gallery-prebuilt.yml. +# +# A job that builds the application without it fails in compile-unity, asking +# for the .NET SDK. + +inputs: + maven-repo-local: + description: The local Maven repository the job builds against. Empty for ~/.m2/repository. + required: false + default: '' + +runs: + using: composite + steps: + - name: Download the Unity gallery prebuilt + uses: ./.github/actions/download-artifact-with-retry + with: + name: unity-gallery-prebuilt + path: ${{ runner.temp }}/unity-gallery-prebuilt + + # bash on every runner, Git Bash on Windows: the script is bash. The + # archive is named through RUNNER_TEMP rather than the expression above so + # that a Windows path is never pasted into the script text, where its + # backslashes would be escapes. + - name: Unpack the Unity gallery prebuilt + shell: bash + env: + UNITY_GALLERY_MAVEN_REPO_LOCAL: ${{ inputs.maven-repo-local }} + run: | + set -euo pipefail + if [ -n "$UNITY_GALLERY_MAVEN_REPO_LOCAL" ]; then + export MAVEN_REPO_LOCAL="$UNITY_GALLERY_MAVEN_REPO_LOCAL" + fi + bash scripts/unity-gallery-prebuilt.sh unpack "$RUNNER_TEMP/unity-gallery-prebuilt/unity-gallery-prebuilt.tgz" diff --git a/.github/ci/apple-checks.json b/.github/ci/apple-checks.json index 9a851f2ef3f..ec63e4e97c1 100644 --- a/.github/ci/apple-checks.json +++ b/.github/ci/apple-checks.json @@ -44,7 +44,15 @@ "!maven/cn1-unity-levelplay/**", "!maven/core-unittests/**", "scripts/hellocodenameone/backend/**", - "scripts/lib/cn1ss.sh" + "scripts/lib/cn1ss.sh", + "scripts/unity-compat-samples/gallery/**", + "maven/cil-translator/**", + "maven/unity-compat/**", + "maven/build-engine/src/main/java/com/codename1/maven/Unity*", + "maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*", + "scripts/unity-gallery-prebuilt.sh", + ".github/workflows/unity-gallery-prebuilt.yml", + ".github/actions/unity-gallery-prebuilt/**" ], "push": [ ".github/workflows/scripts-ios.yml", @@ -90,7 +98,15 @@ "!maven/cn1-unity-levelplay/**", "!maven/core-unittests/**", "scripts/hellocodenameone/backend/**", - "scripts/lib/cn1ss.sh" + "scripts/lib/cn1ss.sh", + "scripts/unity-compat-samples/gallery/**", + "maven/cil-translator/**", + "maven/unity-compat/**", + "maven/build-engine/src/main/java/com/codename1/maven/Unity*", + "maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*", + "scripts/unity-gallery-prebuilt.sh", + ".github/workflows/unity-gallery-prebuilt.yml", + ".github/actions/unity-gallery-prebuilt/**" ] }, "native": { @@ -125,7 +141,15 @@ "!maven/cn1-admob/**", "!maven/cn1-applovin/**", "!maven/cn1-unity-levelplay/**", - "!maven/core-unittests/**" + "!maven/core-unittests/**", + "scripts/unity-compat-samples/gallery/**", + "maven/cil-translator/**", + "maven/unity-compat/**", + "maven/build-engine/src/main/java/com/codename1/maven/Unity*", + "maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*", + "scripts/unity-gallery-prebuilt.sh", + ".github/workflows/unity-gallery-prebuilt.yml", + ".github/actions/unity-gallery-prebuilt/**" ], "push": [ ".github/workflows/scripts-ios-native.yml", @@ -158,7 +182,15 @@ "!maven/cn1-admob/**", "!maven/cn1-applovin/**", "!maven/cn1-unity-levelplay/**", - "!maven/core-unittests/**" + "!maven/core-unittests/**", + "scripts/unity-compat-samples/gallery/**", + "maven/cil-translator/**", + "maven/unity-compat/**", + "maven/build-engine/src/main/java/com/codename1/maven/Unity*", + "maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*", + "scripts/unity-gallery-prebuilt.sh", + ".github/workflows/unity-gallery-prebuilt.yml", + ".github/actions/unity-gallery-prebuilt/**" ] }, "packaging": { @@ -176,10 +208,17 @@ "scripts/check-ios-private-api.py", "scripts/check-ios-sdk-deltas.py", "scripts/run-ios-ui-tests.sh", - "scripts/run-ios-native-tests.sh", "scripts/ios/**", "scripts/hellocodenameone/**", - "!docs/**" + "!docs/**", + "scripts/unity-compat-samples/gallery/**", + "maven/cil-translator/**", + "maven/unity-compat/**", + "maven/build-engine/src/main/java/com/codename1/maven/Unity*", + "maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*", + "scripts/unity-gallery-prebuilt.sh", + ".github/workflows/unity-gallery-prebuilt.yml", + ".github/actions/unity-gallery-prebuilt/**" ], "push": [ ".github/workflows/ios-packaging.yml", @@ -195,10 +234,17 @@ "scripts/check-ios-private-api.py", "scripts/check-ios-sdk-deltas.py", "scripts/run-ios-ui-tests.sh", - "scripts/run-ios-native-tests.sh", "scripts/ios/**", "scripts/hellocodenameone/**", - "!docs/**" + "!docs/**", + "scripts/unity-compat-samples/gallery/**", + "maven/cil-translator/**", + "maven/unity-compat/**", + "maven/build-engine/src/main/java/com/codename1/maven/Unity*", + "maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*", + "scripts/unity-gallery-prebuilt.sh", + ".github/workflows/unity-gallery-prebuilt.yml", + ".github/actions/unity-gallery-prebuilt/**" ] }, "catalyst": { @@ -234,7 +280,15 @@ "!maven/cn1-unity-levelplay/**", "!maven/core-unittests/**", "!docs/**", - "scripts/hellocodenameone/backend/**" + "scripts/hellocodenameone/backend/**", + "scripts/unity-compat-samples/gallery/**", + "maven/cil-translator/**", + "maven/unity-compat/**", + "maven/build-engine/src/main/java/com/codename1/maven/Unity*", + "maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*", + "scripts/unity-gallery-prebuilt.sh", + ".github/workflows/unity-gallery-prebuilt.yml", + ".github/actions/unity-gallery-prebuilt/**" ], "push": [ ".github/workflows/scripts-mac-catalyst.yml", @@ -262,7 +316,15 @@ "!maven/cn1-applovin/**", "!maven/cn1-unity-levelplay/**", "!maven/core-unittests/**", - "scripts/hellocodenameone/backend/**" + "scripts/hellocodenameone/backend/**", + "scripts/unity-compat-samples/gallery/**", + "maven/cil-translator/**", + "maven/unity-compat/**", + "maven/build-engine/src/main/java/com/codename1/maven/Unity*", + "maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*", + "scripts/unity-gallery-prebuilt.sh", + ".github/workflows/unity-gallery-prebuilt.yml", + ".github/actions/unity-gallery-prebuilt/**" ] } } diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml index 07a7ba8cc87..9025b14200f 100644 --- a/.github/codeql/codeql-config.yml +++ b/.github/codeql/codeql-config.yml @@ -16,3 +16,8 @@ paths-ignore: # casts -- so every program there must compile and behave exactly as javac's # output does; "fixing" what CodeQL flags would remove the case under test. - 'vm/JavaCompiler/tests/**' + # The JavaScript target's numeric differential fixture, for the same reason: + # it compares narrowing compound assignments, saturating casts and overflow + # against the JVM, and writing the casts out would change the bytecode shape + # (the DUP2 of a compound array store) that the comparison is there to cover. + - 'vm/tests/src/test/resources/com/codename1/tools/translator/JsNumericDifferentialApp.java' diff --git a/.github/workflows/_build-ios-port.yml b/.github/workflows/_build-ios-port.yml index 81c18031a81..8b857c698ea 100644 --- a/.github/workflows/_build-ios-port.yml +++ b/.github/workflows/_build-ios-port.yml @@ -8,6 +8,11 @@ name: _Build iOS port (reusable) on: workflow_call: + inputs: + test_server: + description: "Also build the CN1SS test server and hand it over beside the bundle. For callers whose jobs run a screenshot suite." + type: boolean + default: false outputs: cn1_built_cache_key: description: "Cache key the build job used for the cn1-built artifact bundle. Dependent jobs should reuse this exact key to avoid recomputing the hash and getting a spurious cache miss." @@ -191,10 +196,57 @@ jobs: "${RUNNER_TEMP#/}/codenameone-tools" ls -la "$RUNNER_TEMP/cn1-built-bundle.tar" + # The CI test server (scripts/hellocodenameone/backend) as a runnable copy, + # the way linux-build-run.yml and the Windows workflows hand theirs over: + # `server.sh dist` here, CN1SS_SERVER_DIST in the jobs that run a suite. + # + # Every screenshot job used to build it for itself, at the top of its + # test step: install the backend runtime, the build engine and the + # plugin from this checkout, then compile the server. Measured, that was + # 3 to 3.6 minutes in each macos-15 job and 6.5 to 9.7 minutes on the + # Xcode 27 leg, whose runner is slower and whose 95 minute limit it was + # spending -- five or six builds of the same classes per run. Built once + # here it costs this job the 3.5 minutes one of them paid. + # + # After the bundle, on purpose: the install below rewrites the plugin and + # the backend jars in ~/.m2, and the bundle is what the application + # builds read. It is the tree the consumers always restored. + # + # A second tar in the same artifact, not an entry appended to the first, + # so nothing done here can damage the handoff every job depends on. The + # server is JVM classes and jars (the SQLite driver carries every + # platform's library), so the copy runs anywhere; a consumer runs it on + # the JAVA17_HOME this job built it with, which is in the bundle. + # + # Never fatal. A job that finds no copy builds its own, exactly as it did + # before (cn1ss_ws_prepare in scripts/lib/cn1ss.sh), so a failure here + # costs the minutes back and nothing else. + - name: Build the test server + if: ${{ inputs.test_server }} + continue-on-error: true + timeout-minutes: 25 + run: | + set -euo pipefail + source "$RUNNER_TEMP/codenameone-tools/tools/env.sh" + DIST="$RUNNER_TEMP/cn1ss-server" + if RETRY_ATTEMPTS=2 bash scripts/ci/retry.sh \ + scripts/hellocodenameone/backend/server.sh dist "$DIST" \ + && test -f "$DIST/classes/META-INF/cn1-backend-main"; then + tar -C / -cf "$RUNNER_TEMP/cn1ss-server.tar.partial" "${DIST#/}" + mv "$RUNNER_TEMP/cn1ss-server.tar.partial" "$RUNNER_TEMP/cn1ss-server.tar" + ls -la "$RUNNER_TEMP/cn1ss-server.tar" + else + rm -f "$RUNNER_TEMP/cn1ss-server.tar" + echo "::warning::The test server did not build here; each suite job will build its own." + fi + - name: Upload built CN1 + iOS port bundle uses: actions/upload-artifact@v4 with: name: cn1-built-bundle - path: ${{ runner.temp }}/cn1-built-bundle.tar + # cn1ss-server.tar is there only when the step above produced it. + path: | + ${{ runner.temp }}/cn1-built-bundle.tar + ${{ runner.temp }}/cn1ss-server.tar retention-days: 1 if-no-files-found: error diff --git a/.github/workflows/car-android.yml b/.github/workflows/car-android.yml index 339ab703639..33a19b2588e 100644 --- a/.github/workflows/car-android.yml +++ b/.github/workflows/car-android.yml @@ -8,6 +8,16 @@ name: Car (Android Auto) integration 'on': pull_request: paths: + # The application these jobs build compiles a Unity project, from an + # archive that unity-gallery-prebuilt.yml makes out of these. + - 'scripts/unity-compat-samples/gallery/**' + - 'maven/cil-translator/**' + - 'maven/unity-compat/**' + - 'maven/build-engine/src/main/java/com/codename1/maven/Unity*' + - 'maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*' + - 'scripts/unity-gallery-prebuilt.sh' + - '.github/workflows/unity-gallery-prebuilt.yml' + - '.github/actions/unity-gallery-prebuilt/**' - '.github/workflows/car-android.yml' - 'CodenameOne/src/com/codename1/car/**' - 'Ports/Android/src/com/codename1/impl/android/AndroidCarSupport.java' @@ -31,6 +41,16 @@ name: Car (Android Auto) integration branches: - master paths: + # The application these jobs build compiles a Unity project, from an + # archive that unity-gallery-prebuilt.yml makes out of these. + - 'scripts/unity-compat-samples/gallery/**' + - 'maven/cil-translator/**' + - 'maven/unity-compat/**' + - 'maven/build-engine/src/main/java/com/codename1/maven/Unity*' + - 'maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*' + - 'scripts/unity-gallery-prebuilt.sh' + - '.github/workflows/unity-gallery-prebuilt.yml' + - '.github/actions/unity-gallery-prebuilt/**' - '.github/workflows/car-android.yml' - 'CodenameOne/src/com/codename1/car/**' - 'Ports/Android/src/com/codename1/impl/android/AndroidCarSupport.java' @@ -58,7 +78,14 @@ concurrency: cancel-in-progress: true jobs: + # The application has a Unity project in it. One Linux job compiles it and + # hands the result to the jobs below as an artifact; none of them installs + # the .NET SDK or builds the Unity modules. + unity-gallery: + uses: ./.github/workflows/unity-gallery-prebuilt.yml + car-android: + needs: unity-gallery name: Car Android Auto permissions: contents: read @@ -109,6 +136,11 @@ jobs: run: ./scripts/setup-workspace.sh -q -DskipTests - name: Build Android port run: ./scripts/build-android-port.sh -q -DskipTests + # After everything that fills the local Maven repository and before the + # application is built; this job has no .NET SDK (see the action). + - name: Unpack the Unity gallery prebuilt + uses: ./.github/actions/unity-gallery-prebuilt + - name: Build Hello Codename One Android app (car-enabled) id: build-android-app run: | diff --git a/.github/workflows/car-ios.yml b/.github/workflows/car-ios.yml index e319f6a6af4..caf083dcf16 100644 --- a/.github/workflows/car-ios.yml +++ b/.github/workflows/car-ios.yml @@ -9,6 +9,16 @@ name: Car (CarPlay) integration 'on': pull_request: paths: + # The application these jobs build compiles a Unity project, from an + # archive that unity-gallery-prebuilt.yml makes out of these. + - 'scripts/unity-compat-samples/gallery/**' + - 'maven/cil-translator/**' + - 'maven/unity-compat/**' + - 'maven/build-engine/src/main/java/com/codename1/maven/Unity*' + - 'maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*' + - 'scripts/unity-gallery-prebuilt.sh' + - '.github/workflows/unity-gallery-prebuilt.yml' + - '.github/actions/unity-gallery-prebuilt/**' - '.github/workflows/car-ios.yml' - 'CodenameOne/src/com/codename1/car/**' - 'Ports/iOSPort/src/com/codename1/impl/ios/IOSCarBridge.java' @@ -22,6 +32,16 @@ name: Car (CarPlay) integration branches: - master paths: + # The application these jobs build compiles a Unity project, from an + # archive that unity-gallery-prebuilt.yml makes out of these. + - 'scripts/unity-compat-samples/gallery/**' + - 'maven/cil-translator/**' + - 'maven/unity-compat/**' + - 'maven/build-engine/src/main/java/com/codename1/maven/Unity*' + - 'maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*' + - 'scripts/unity-gallery-prebuilt.sh' + - '.github/workflows/unity-gallery-prebuilt.yml' + - '.github/actions/unity-gallery-prebuilt/**' - '.github/workflows/car-ios.yml' - 'CodenameOne/src/com/codename1/car/**' - 'Ports/iOSPort/src/com/codename1/impl/ios/IOSCarBridge.java' @@ -39,7 +59,14 @@ concurrency: cancel-in-progress: true jobs: + # The application has a Unity project in it. One Linux job compiles it and + # hands the result to the jobs below as an artifact; none of them installs + # the .NET SDK or builds the Unity modules. + unity-gallery: + uses: ./.github/workflows/unity-gallery-prebuilt.yml + car-ios: + needs: unity-gallery name: Car CarPlay permissions: contents: read @@ -85,6 +112,11 @@ jobs: run: ./scripts/setup-workspace.sh -q -DskipTests - name: Build iOS port run: ./scripts/build-ios-port.sh -q -DskipTests + # After everything that fills the local Maven repository and before the + # application is built; this job has no .NET SDK (see the action). + - name: Unpack the Unity gallery prebuilt + uses: ./.github/actions/unity-gallery-prebuilt + - name: Build sample iOS app (car-enabled) and compile workspace id: build-ios-app run: ./scripts/build-ios-app.sh -q -DskipTests diff --git a/.github/workflows/health-android.yml b/.github/workflows/health-android.yml index 0398e33e53d..3a4c7bca2db 100644 --- a/.github/workflows/health-android.yml +++ b/.github/workflows/health-android.yml @@ -15,6 +15,16 @@ name: Health (Health Connect) integration 'on': pull_request: paths: + # The application these jobs build compiles a Unity project, from an + # archive that unity-gallery-prebuilt.yml makes out of these. + - 'scripts/unity-compat-samples/gallery/**' + - 'maven/cil-translator/**' + - 'maven/unity-compat/**' + - 'maven/build-engine/src/main/java/com/codename1/maven/Unity*' + - 'maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*' + - 'scripts/unity-gallery-prebuilt.sh' + - '.github/workflows/unity-gallery-prebuilt.yml' + - '.github/actions/unity-gallery-prebuilt/**' - '.github/workflows/health-android.yml' - 'CodenameOne/src/com/codename1/health/**' - 'CodenameOne/src/com/codename1/impl/health/**' @@ -30,6 +40,16 @@ name: Health (Health Connect) integration branches: - master paths: + # The application these jobs build compiles a Unity project, from an + # archive that unity-gallery-prebuilt.yml makes out of these. + - 'scripts/unity-compat-samples/gallery/**' + - 'maven/cil-translator/**' + - 'maven/unity-compat/**' + - 'maven/build-engine/src/main/java/com/codename1/maven/Unity*' + - 'maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*' + - 'scripts/unity-gallery-prebuilt.sh' + - '.github/workflows/unity-gallery-prebuilt.yml' + - '.github/actions/unity-gallery-prebuilt/**' - '.github/workflows/health-android.yml' - 'CodenameOne/src/com/codename1/health/**' - 'CodenameOne/src/com/codename1/impl/health/**' @@ -49,7 +69,14 @@ concurrency: cancel-in-progress: true jobs: + # The application has a Unity project in it. One Linux job compiles it and + # hands the result to the jobs below as an artifact; none of them installs + # the .NET SDK or builds the Unity modules. + unity-gallery: + uses: ./.github/workflows/unity-gallery-prebuilt.yml + health-android: + needs: unity-gallery name: Health Connect Android permissions: contents: read @@ -88,6 +115,11 @@ jobs: run: ./scripts/setup-workspace.sh -q -DskipTests - name: Build Android port run: ./scripts/build-android-port.sh -q -DskipTests + # After everything that fills the local Maven repository and before the + # application is built; this job has no .NET SDK (see the action). + - name: Unpack the Unity gallery prebuilt + uses: ./.github/actions/unity-gallery-prebuilt + - name: Make the smoke app a health app run: | set -euo pipefail diff --git a/.github/workflows/ios-packaging.yml b/.github/workflows/ios-packaging.yml index d22e99c8574..d4f349c469f 100644 --- a/.github/workflows/ios-packaging.yml +++ b/.github/workflows/ios-packaging.yml @@ -8,6 +8,11 @@ on: port_prepared: type: boolean default: false + # True when the caller already ran unity-gallery-prebuilt.yml in this run: + # its artifact has a fixed name and cannot be uploaded twice. + unity_prepared: + type: boolean + default: false concurrency: # Only the latest revision needs validation. PR numbers keep forks isolated; @@ -16,21 +21,31 @@ concurrency: cancel-in-progress: true jobs: + # The application has a Unity project in it. One Linux job compiles it and + # hands the result to the job below as an artifact, unless the caller did. + unity-gallery: + if: ${{ !inputs.unity_prepared }} + uses: ./.github/workflows/unity-gallery-prebuilt.yml + build-port: if: ${{ !inputs.port_prepared }} uses: ./.github/workflows/_build-ios-port.yml + with: + test_server: true packaging: - if: ${{ !cancelled() && (inputs.port_prepared || needs.build-port.result == 'success') }} - needs: build-port + if: ${{ !cancelled() && (inputs.port_prepared || needs.build-port.result == 'success') && (inputs.unity_prepared || needs.unity-gallery.result == 'success') }} + needs: [build-port, unity-gallery] permissions: contents: read runs-on: macos-15 - # This job runs device Release, simulator UI, and notification builds in - # sequence. On a cold runner all tests passed after 119m44s, leaving only - # 16s of the old two-hour budget for artifact upload and cache saves. Keep + # This job runs device Release and simulator UI builds in sequence. The + # numbers below are from when a third build, for the notification tests, + # followed them; it is gone (see the end of the job) and the limit has been + # left where it was. On a cold runner all tests passed after 119m44s, leaving + # only 16s of the old two-hour budget for artifact upload and cache saves. Keep # the individual step limits below and allow 30m of finalization headroom. - # The hello app now also translates the Android compatibility gallery, its runtime and the Kotlin stdlib; 150 no longer covers the three builds. + # The hello app now also translates the Android compatibility gallery, its runtime and the Kotlin stdlib; 150 no longer covered the three builds. timeout-minutes: 210 # Exercises both CocoaPods and SPM in a single Xcode project. Catches # regressions in either dependency manager (each pathway runs end to end) @@ -138,6 +153,22 @@ jobs: # Replaces the evictable cn1-built cache restore with the artifact # build-port handed off (not LRU-evicted, survives --failed re-runs). tar -C / -xf "$RUNNER_TEMP/cn1-built-bundle.tar" + # The test server build-port built for the whole run, when it did. + # Without it the suite script builds its own, as it always has. + if [ -f "$RUNNER_TEMP/cn1ss-server.tar" ]; then + tar -C / -xf "$RUNNER_TEMP/cn1ss-server.tar" + fi + if [ -d "$RUNNER_TEMP/cn1ss-server/classes" ]; then + echo "CN1SS_SERVER_DIST=$RUNNER_TEMP/cn1ss-server" >> "$GITHUB_ENV" + echo "Using the prebuilt test server in $RUNNER_TEMP/cn1ss-server" + else + echo "No prebuilt test server in the bundle; the suite builds its own." + fi + + # After everything that fills the local Maven repository and before the + # application is built; this job has no .NET SDK (see the action). + - name: Unpack the Unity gallery prebuilt + uses: ./.github/actions/unity-gallery-prebuilt - name: Build sample iOS app id: build_ios_app @@ -209,8 +240,8 @@ jobs: # within budget after the gaming runtime enlarged core. The benchmark/metal pipeline # keeps the -O2 default. CN1_TEST_OPT_LEVEL: "1" - # Shared, cached derived-data dir: the native-test step reuses these compiled objects, - # and a re-run restores them, instead of recompiling the whole translated core. + # Cached derived-data dir: a re-run restores these compiled objects instead of + # recompiling the whole translated core. CN1_IOS_DERIVED_DATA: ${{ runner.temp }}/cn1-ios-derived run: | set -euo pipefail @@ -224,34 +255,21 @@ jobs: # The cold -O1 compile alone reached 52 minutes once the hello app gained the Android compatibility gallery, its runtime and the Kotlin stdlib. timeout-minutes: 100 - - name: Run native iOS notification tests - env: - ARTIFACTS_DIR: ${{ github.workspace }}/artifacts/ios-packaging-native - # Reuse the app + core already compiled by the UI-smoke step (same shared derived data) - # so this test build does not recompile the whole translated core a second time. - CN1_IOS_DERIVED_DATA: ${{ runner.temp }}/cn1-ios-derived - CN1_TEST_OPT_LEVEL: "1" - run: | - set -euo pipefail - mkdir -p "${ARTIFACTS_DIR}" - ./scripts/run-ios-native-tests.sh \ - "${{ steps.build_ios_app.outputs.workspace }}" \ - "${{ steps.build_ios_app.outputs.scheme }}" - # Two things grew into this budget, and both still apply. The reuse above is partial: - # run-ios-native-tests.sh injects the test target into the pbxproj before building, and - # Xcode rebuilds the app target after a project change even though the objects are there, - # so this step still pays a near-full native build -- 6m55s of the 8m13s it takes on a - # fast runner. Measured on one commit, a slow runner spent 4m31s booting the simulator - # (a fast one: 1m15s) and 13 minutes on that build, and the cap fired before the tests - # started. On top of that the sample now declares a watchMain, so the app scheme carries - # an embedded watch app with a ParparVM translation of its own, which xcodebuild builds - # before running a single test -- the raise scripts-ios-native.yml already took. - # - # So the higher of the two, not either alone: the watch build is added work on top of the - # rebuild, not a re-description of it. The real fix is to inject before the UI-smoke step - # so this build is genuinely incremental; until then the budget has to cover the build it - # actually does. The hello app's growth (the Android compatibility gallery, its runtime and the Kotlin stdlib) added roughly half again to that build. - timeout-minutes: 60 + # No XCTest run here. This job used to end with run-ios-native-tests.sh, the + # same notification tests scripts-ios-native.yml runs, from the same script. + # Measured over three runs it took 23.5, 25.3 and 38 minutes of this job, and + # the tests in it take 0.17 seconds: the rest was the rebuild Xcode does after + # the test target is injected into the project. + # + # scripts-ios-native.yml is the one that stays. Both are called from + # scripts-ios.yml on the same events, and .github/ci/apple-checks.json selects + # that suite for every path that selected this one for the tests' sake, plus + # the core, the VM and the Maven modules, which this job is not selected for. + # What is no longer covered anywhere: the tests built in THIS job's + # configuration -- CocoaPods and SPM together, Google sign-in on, -O1. The + # application itself still builds and runs that way in the smoke above. + # A standalone dispatch of this workflow runs no XCTest; dispatch + # scripts-ios-native.yml for that. - name: Upload packaging artifacts if: ${{ !cancelled() }} diff --git a/.github/workflows/linux-build-run.yml b/.github/workflows/linux-build-run.yml index 7072a1ca1df..76ac7285635 100644 --- a/.github/workflows/linux-build-run.yml +++ b/.github/workflows/linux-build-run.yml @@ -15,7 +15,8 @@ # # Jobs: # prepare-suite (x64) -> builds the plugin + hellocodenameone-common, -# uploads the suite + ads-mock classes +# uploads the suite classes and those of +# the libraries it ships (translation-libs) # build-run (matrix x64 + arm64) -> core + Linux port, translate + build + run, # uploads PNGs/arch # compare-comment (ubuntu) -> diffs each arch vs its in-repo baseline, @@ -31,6 +32,16 @@ on: - cron: '45 1 * * *' pull_request: paths: + # The application these jobs build compiles a Unity project, from an + # archive that unity-gallery-prebuilt.yml makes out of these. + - 'scripts/unity-compat-samples/gallery/**' + - 'maven/cil-translator/**' + - 'maven/unity-compat/**' + - 'maven/build-engine/src/main/java/com/codename1/maven/Unity*' + - 'maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*' + - 'scripts/unity-gallery-prebuilt.sh' + - '.github/workflows/unity-gallery-prebuilt.yml' + - '.github/actions/unity-gallery-prebuilt/**' - '.github/workflows/linux-build-run.yml' # The package install goes through these, and they run from this workflow # too -- a change to either must be exercised by the job that depends on @@ -60,6 +71,16 @@ on: push: branches: [ master, main ] paths: + # The application these jobs build compiles a Unity project, from an + # archive that unity-gallery-prebuilt.yml makes out of these. + - 'scripts/unity-compat-samples/gallery/**' + - 'maven/cil-translator/**' + - 'maven/unity-compat/**' + - 'maven/build-engine/src/main/java/com/codename1/maven/Unity*' + - 'maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*' + - 'scripts/unity-gallery-prebuilt.sh' + - '.github/workflows/unity-gallery-prebuilt.yml' + - '.github/actions/unity-gallery-prebuilt/**' - '.github/workflows/linux-build-run.yml' - 'scripts/ci/apt-get-install.sh' - 'scripts/ci/apt-get-update.sh' @@ -88,11 +109,19 @@ concurrency: cancel-in-progress: true jobs: + # The application has a Unity project in it. One Linux job compiles it and + # hands the result to the jobs below as an artifact; none of them installs + # the .NET SDK or builds the Unity modules. + unity-gallery: + uses: ./.github/workflows/unity-gallery-prebuilt.yml + # Build the codenameone maven plugin (it needs CEF, which is x64-only on Linux) - # and the hellocodenameone screenshot suite, then publish the suite + ads-mock - # *.class. These are architecture-independent (they are translated to C later), + # and the hellocodenameone screenshot suite, then publish the suite *.class + # and the *.class of the libraries it ships (common/target/translation-libs: + # the mock ad provider, the Unity runtime). These are architecture-independent (they are translated to C later), # so both arch legs translate from these same classes. prepare-suite: + needs: unity-gallery name: build suite classes (x64) runs-on: ubuntu-24.04 timeout-minutes: 60 @@ -126,6 +155,11 @@ jobs: mvn -B -pl codenameone-maven-plugin,cn1-ads-mock,backend,backend-test -am \ -DskipTests '-Dmaven.javadoc.skip=true' '-Plocal-dev-javase' install + # After everything that fills the local Maven repository and before the + # application is built; this job has no .NET SDK (see the action). + - name: Unpack the Unity gallery prebuilt + uses: ./.github/actions/unity-gallery-prebuilt + - name: Build hellocodenameone-common (JDK 17) run: | set -e @@ -153,7 +187,7 @@ jobs: set -e tar czf suite-classes.tgz \ scripts/hellocodenameone/common/target/classes \ - maven/cn1-ads-mock/target/classes \ + scripts/hellocodenameone/common/target/translation-libs \ artifacts/cn1ss-server - name: Upload suite classes uses: actions/upload-artifact@v4 diff --git a/.github/workflows/parparvm-tests-windows.yml b/.github/workflows/parparvm-tests-windows.yml index 6a880ffc4ab..5fe3893d0d7 100644 --- a/.github/workflows/parparvm-tests-windows.yml +++ b/.github/workflows/parparvm-tests-windows.yml @@ -25,6 +25,16 @@ on: - cron: '5 2 * * *' pull_request: paths: + # The application these jobs build compiles a Unity project, from an + # archive that unity-gallery-prebuilt.yml makes out of these. + - 'scripts/unity-compat-samples/gallery/**' + - 'maven/cil-translator/**' + - 'maven/unity-compat/**' + - 'maven/build-engine/src/main/java/com/codename1/maven/Unity*' + - 'maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*' + - 'scripts/unity-gallery-prebuilt.sh' + - '.github/workflows/unity-gallery-prebuilt.yml' + - '.github/actions/unity-gallery-prebuilt/**' - '.github/workflows/parparvm-tests-windows.yml' - 'vm/**' - 'Ports/WindowsPort/**' @@ -48,6 +58,16 @@ on: push: branches: [ master, main ] paths: + # The application these jobs build compiles a Unity project, from an + # archive that unity-gallery-prebuilt.yml makes out of these. + - 'scripts/unity-compat-samples/gallery/**' + - 'maven/cil-translator/**' + - 'maven/unity-compat/**' + - 'maven/build-engine/src/main/java/com/codename1/maven/Unity*' + - 'maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*' + - 'scripts/unity-gallery-prebuilt.sh' + - '.github/workflows/unity-gallery-prebuilt.yml' + - '.github/actions/unity-gallery-prebuilt/**' - '.github/workflows/parparvm-tests-windows.yml' - 'vm/**' - 'Ports/WindowsPort/**' @@ -83,6 +103,12 @@ env: CN1_NATIVE_VERIFY: strict jobs: + # The application has a Unity project in it. One Linux job compiles it and + # hands the result to the jobs below as an artifact; none of them installs + # the .NET SDK or builds the Unity modules. + unity-gallery: + uses: ./.github/workflows/unity-gallery-prebuilt.yml + vm-tests-windows: name: clean-target (${{ matrix.name }}) runs-on: ${{ matrix.runner }} @@ -308,6 +334,7 @@ jobs: # Separate from the clean-target gate so rendering failures are reported as # their own blocking check with dedicated diagnostics. windows-port-screenshot: + needs: unity-gallery name: screenshot-capture (x64) if: github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' runs-on: windows-latest @@ -399,6 +426,11 @@ jobs: } if (-not $ok) { throw "mvn install failed after all retries" } + # After everything that fills the local Maven repository and before the + # application is built; this job has no .NET SDK (see the action). + - name: Unpack the Unity gallery prebuilt + uses: ./.github/actions/unity-gallery-prebuilt + # Build the hellocodenameone screenshot app into common/target/classes, which # the capture test translates into the native exe. This is a plain Maven build # of the sample on JDK 17 -- exactly what a fresh initializr project does -- so @@ -437,12 +469,21 @@ jobs: $out = "scripts/hellocodenameone/common/target/classes" if (!(Test-Path "$out/com/codenameone/examples/hellocodenameone/tests/Cn1ssDeviceRunner.class")) { throw "common classes missing" } if (!(Test-Path "$out/com/codename1/generated/svg/SVGRegistry.class")) { throw "transcode-svg did not produce SVGRegistry" } + # The libraries the application ships (stage-translation-libs in its pom): + # the capture test hands them to the translator as a source root. + $libs = "scripts/hellocodenameone/common/target/translation-libs" + if (!(Test-Path "$libs/com/codename1/ads/mock/MockAdProvider.class")) { throw "translation-libs has no mock ad provider" } + if (!(Test-Path "$libs/com/codename1/unitycompat/unityengine/UnityRuntime.class")) { throw "translation-libs has no Unity runtime" } + # Two paths, so the artifact is rooted at their common parent, + # common/target -- which is where the arm64 leg restores it. - name: Upload hellocodenameone-common classes uses: ./.github/actions/upload-artifact-with-retry with: name: windows-port-common-classes - path: scripts/hellocodenameone/common/target/classes + path: | + scripts/hellocodenameone/common/target/classes + scripts/hellocodenameone/common/target/translation-libs if-no-files-found: error retention-days: 1 @@ -483,7 +524,6 @@ jobs: path: | maven/core/target/classes maven/windows/target/classes - maven/cn1-ads-mock/target/classes if-no-files-found: error retention-days: 1 @@ -693,7 +733,7 @@ jobs: uses: ./.github/actions/download-artifact-with-retry with: name: windows-port-common-classes - path: scripts/hellocodenameone/common/target/classes + path: scripts/hellocodenameone/common/target # Fail loudly if the artifacts did not land where the test reads them -- # otherwise the capture test would silently skip (assumeTrue) and post nothing. @@ -706,6 +746,10 @@ jobs: if (!(Test-Path $core)) { Write-Error "core classes missing after download: $core"; exit 1 } if (!(Test-Path $port)) { Write-Error "port classes missing after download: $port"; exit 1 } if (!(Test-Path $common)) { Write-Error "common classes missing after download: $common"; exit 1 } + $libs = "scripts/hellocodenameone/common/target/translation-libs" + foreach ($lib in "com/codename1/ads/mock/MockAdProvider.class", "com/codename1/unitycompat/unityengine/UnityRuntime.class") { + if (!(Test-Path "$libs/$lib")) { Write-Error "application libraries missing after download: $libs/$lib"; exit 1 } + } Write-Host "Restored core + Windows port + common classes OK" # Fetch the WebView2 SDK so buildHelloCodenameOneExe's CMake compiles the diff --git a/.github/workflows/parparvm-tests.yml b/.github/workflows/parparvm-tests.yml index f24feaba7d8..800708c2c9b 100644 --- a/.github/workflows/parparvm-tests.yml +++ b/.github/workflows/parparvm-tests.yml @@ -7,6 +7,17 @@ on: - '.github/workflows/parparvm-tests.yml' - 'vm/**' - 'Ports/JavaScriptPort/**' + # GpuBlendContractTest reads each GPU backend's blend factors out of its + # source; a change to one of these alone must still run it. + - 'Ports/LinuxPort/nativeSources/cn1_linux_gl.c' + - 'Ports/iOSPort/nativeSources/CN1GL3D.m' + - 'Ports/WindowsPort/nativeSources/cn1_windows_d3d.cpp' + - 'Ports/Android/src/com/codename1/impl/android/AndroidGraphicsDevice.java' + - 'Ports/JavaSE/src/com/codename1/impl/javase/JavaSEGLDevice.java' + # NativeFontStyleContractTest compiles the function that reads a native: + # font name out of each of these and reads where the style goes. + - 'Ports/LinuxPort/nativeSources/cn1_linux_text.c' + - 'Ports/WindowsPort/nativeSources/cn1_windows_text.c' # The GENERATOR, because the translated backend tests run what it writes: # a change to the @Entity processor or BackendPackageMojo alone reaches # PostgreSQL and MySQL through the daos it generates, and this is the only @@ -60,6 +71,17 @@ on: - '.github/workflows/parparvm-tests.yml' - 'vm/**' - 'Ports/JavaScriptPort/**' + # GpuBlendContractTest reads each GPU backend's blend factors out of its + # source; a change to one of these alone must still run it. + - 'Ports/LinuxPort/nativeSources/cn1_linux_gl.c' + - 'Ports/iOSPort/nativeSources/CN1GL3D.m' + - 'Ports/WindowsPort/nativeSources/cn1_windows_d3d.cpp' + - 'Ports/Android/src/com/codename1/impl/android/AndroidGraphicsDevice.java' + - 'Ports/JavaSE/src/com/codename1/impl/javase/JavaSEGLDevice.java' + # NativeFontStyleContractTest compiles the function that reads a native: + # font name out of each of these and reads where the style goes. + - 'Ports/LinuxPort/nativeSources/cn1_linux_text.c' + - 'Ports/WindowsPort/nativeSources/cn1_windows_text.c' # The GENERATOR, because the translated backend tests run what it writes: # a change to the @Entity processor or BackendPackageMojo alone reaches # PostgreSQL and MySQL through the daos it generates, and this is the only diff --git a/.github/workflows/port-status-nightly.yml b/.github/workflows/port-status-nightly.yml index eebdf41c3d0..15452432b7b 100644 --- a/.github/workflows/port-status-nightly.yml +++ b/.github/workflows/port-status-nightly.yml @@ -11,6 +11,12 @@ permissions: contents: write jobs: + # The application has a Unity project in it. One Linux job compiles it and + # hands the result to the jobs below as an artifact; none of them installs + # the .NET SDK or builds the Unity modules. + unity-gallery: + uses: ./.github/workflows/unity-gallery-prebuilt.yml + # port-status-publish.yml only publishes a report when a workflow_run event # reaches it, and those events never arrive for some producers -- the Linux # and Windows suites had not landed a single report, so the public table @@ -30,6 +36,7 @@ jobs: run: scripts/hellocodenameone/conformance/backfill_port_status.sh build-javascript-app: + needs: unity-gallery runs-on: ubuntu-24.04 timeout-minutes: 60 steps: @@ -51,6 +58,11 @@ jobs: cache: maven - name: Set up workspace run: ./scripts/setup-workspace.sh -q -DskipTests + # After everything that fills the local Maven repository and before the + # application is built; this job has no .NET SDK (see the action). + - name: Unpack the Unity gallery prebuilt + uses: ./.github/actions/unity-gallery-prebuilt + - name: Build compliance application for JavaScript run: SKIP_PARPARVM_BUILD=1 ./scripts/build-javascript-port-hellocodenameone.sh "${GITHUB_WORKSPACE}/hellocodenameone-javascript-port.zip" - uses: actions/upload-artifact@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 896118dd33e..4b1a2c10a47 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,6 +39,14 @@ jobs: R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} R2_BUCKET: ${{ vars.R2_BUCKET }} R2_BASE_URL: https://repo.codenameone.com/maven2 + # The .NET SDK, which the core reactor needs for codenameone-unity-compat (see + # "Set up .NET" below). Its first-run state and the NuGet package cache stay in + # the workspace rather than the runner's home directory, so the one package the + # build restores is visibly this job's and nothing outlives it. + DOTNET_CLI_HOME: ${{ github.workspace }}/.dotnet/cli-home + NUGET_PACKAGES: ${{ github.workspace }}/.nuget-packages + DOTNET_CLI_TELEMETRY_OPTOUT: '1' + DOTNET_NOLOGO: '1' steps: - uses: actions/checkout@v6 @@ -63,9 +71,40 @@ jobs: git config --global user.name "GitHub Actions Bot" git config --global user.email "github-actions@codenameone.com" + # codenameone-cil-translator and codenameone-unity-compat are published by the + # core reactor below, at the release's own version: the archetype's poms and + # both build plugins resolve them at that version, so a release without them + # is one whose generated projects cannot build a Unity source tree. The + # runtime's build compiles two C# projects, hence the SDK. Both target + # netstandard2.1, which any current SDK builds; the channel is the one + # unity-compat.yml tests the same build with, so the release compiles the C# + # with the compiler CI exercised. + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + - name: Check Maven Version run: mvn -v + - name: Require the .NET SDK + # Not continue-on-error, for the reason the frozen-artifact check below is + # not: nothing has been built yet, so there is nothing to salvage. Without + # this the same absence surfaces an hour into the reactor, as exec-maven-plugin + # failing to start a program in a module nobody was looking at. + run: | + if ! command -v dotnet >/dev/null 2>&1; then + echo "The .NET SDK is not on the PATH. The release builds" >&2 + echo "codenameone-unity-compat, which compiles C# with it." >&2 + exit 1 + fi + if [ -z "$(dotnet --list-sdks)" ]; then + echo "dotnet is on the PATH but no SDK is installed (a runtime alone" >&2 + echo "cannot build). The release builds codenameone-unity-compat with it." >&2 + exit 1 + fi + dotnet --version + - name: Verify frozen artifacts are on R2 # A release resolves codenameone-designer at its pinned version but does not # publish it ( in maven/pom.xml). While Central was in every @@ -84,6 +123,24 @@ jobs: run: | cat $HOME/.m2/settings.xml cd maven + # update-version.sh runs `mvn versions:set` with no profile, and it + # does not need -Dunity-compat to reach cil-translator and + # unity-compat: versions:set walks the of every profile of + # the aggregator, active or not, which is also how core-unittests, + # tests and backend-parparvm have always been bumped. Measured, not + # assumed (versions-maven-plugin 2.22.0): on a copy of the pom tree, + # mvn versions:set -DnewVersion=9.9.9-fake && mvn versions:commit + # leaves both modules' and at the new + # version, and nothing else names their version -- unity-compat asks + # for the translator at ${project.version}, and the archetype and + # both build plugins resolve the pair at their own. To check again: + # mkdir /tmp/v && (cd maven && git ls-files | grep 'pom\.xml$' \ + # | cpio -pdm /tmp/v) && (cd /tmp/v && mvn versions:set \ + # -DnewVersion=9.9.9-fake) && grep -c 9.9.9-fake \ + # /tmp/v/cil-translator/pom.xml /tmp/v/unity-compat/pom.xml + # which prints 2 for each. "Confirm artifacts on R2" below names + # both artifacts at the release version, so a release in which this + # had stopped being true would go red there, not publish quietly. bash update-version.sh "$GITHUB_REF_NAME" export GPG_TTY=$(tty) # R2 is the repository. `deploy` is still the phase to run because @@ -99,8 +156,14 @@ jobs: # this same reactor, so at that moment it exists only in the local # repo and no remote can serve it. -DskipTests covers the matching # surefire pass. + # -Dunity-compat: codenameone-cil-translator and codenameone-unity-compat + # sit in a profile, because building the second needs the .NET SDK and + # an ordinary framework build must not. A release does publish them, in + # this pass and at this version, so the profile is on here. Leaving the + # flag off does not fail the build -- the two modules simply drop out + # of the reactor -- which is why "Confirm artifacts on R2" names them. xvfb-run -a mvn deploy -Psign-artifacts -Dgpg.passphrase=$MAVEN_GPG_PASSPHRASE \ - -Plocal-dev-javase -Darchetype.test.skip=true -DskipTests \ + -Plocal-dev-javase -Dunity-compat -Darchetype.test.skip=true -DskipTests \ -DskipPublishing=true env: MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} @@ -138,6 +201,7 @@ jobs: set -e for artifact in codenameone-core codenameone-maven-plugin codenameone-platform-feature-catalog \ codenameone-build-hint-catalog codenameone-build-hint-tools \ + codenameone-cil-translator codenameone-unity-compat \ cn1app-archetype cn1lib-archetype; do url="${R2_BASE_URL}/com/codenameone/${artifact}/${GITHUB_REF_NAME}/${artifact}-${GITHUB_REF_NAME}.pom?cb=${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" code=$(curl -s -o /dev/null -w "%{http_code}" "$url") @@ -147,6 +211,22 @@ jobs: fi echo "ok: $artifact ${GITHUB_REF_NAME}" done + # A pom says the module was staged, not that its jars were. These three are + # resolved by name during an application's build -- the runtime merged into + # the application, the translator forked as a tool, and the "references" + # classifier its C# compiles against -- and the last is an attachment no + # pom mentions, so only asking for the file itself notices it missing. + for file in codenameone-cil-translator/${GITHUB_REF_NAME}/codenameone-cil-translator-${GITHUB_REF_NAME}.jar \ + codenameone-unity-compat/${GITHUB_REF_NAME}/codenameone-unity-compat-${GITHUB_REF_NAME}.jar \ + codenameone-unity-compat/${GITHUB_REF_NAME}/codenameone-unity-compat-${GITHUB_REF_NAME}-references.jar; do + url="${R2_BASE_URL}/com/codenameone/${file}?cb=${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + code=$(curl -s -o /dev/null -w "%{http_code}" "$url") + if [ "$code" != "200" ]; then + echo "MISSING on R2: $file (HTTP $code)" >&2 + exit 1 + fi + echo "ok: $file" + done # --- Game Builder editor ------------------------------------------------- # The cn1:gamebuilder goal resolves @@ -379,7 +459,7 @@ jobs: # poll, not the deploy, was the verdict, and the marker could not trust it. With # skipPublishing there is no upload to misreport: a build that fails, fails, and # the tree it staged may be missing whichever modules came after the failure. - # r2_core_confirm only covers seven key artifacts, so without this a build that + # r2_core_confirm only covers nine key artifacts, so without this a build that # died late could publish and advertise a version that is quietly incomplete. if: >- always() && steps.deploy.outcome == 'success' && diff --git a/.github/workflows/scripts-android.yml b/.github/workflows/scripts-android.yml index 36dac798ffe..3726c551738 100644 --- a/.github/workflows/scripts-android.yml +++ b/.github/workflows/scripts-android.yml @@ -10,6 +10,16 @@ name: Test Android build scripts - cron: '5 1 * * *' pull_request: paths: + # The application these jobs build compiles a Unity project, from an + # archive that unity-gallery-prebuilt.yml makes out of these. + - 'scripts/unity-compat-samples/gallery/**' + - 'maven/cil-translator/**' + - 'maven/unity-compat/**' + - 'maven/build-engine/src/main/java/com/codename1/maven/Unity*' + - 'maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*' + - 'scripts/unity-gallery-prebuilt.sh' + - '.github/workflows/unity-gallery-prebuilt.yml' + - '.github/actions/unity-gallery-prebuilt/**' - '.github/workflows/scripts-android.yml' - 'scripts/setup-workspace.sh' - 'scripts/ci/retry.sh' @@ -57,6 +67,16 @@ name: Test Android build scripts branches: - master paths: + # The application these jobs build compiles a Unity project, from an + # archive that unity-gallery-prebuilt.yml makes out of these. + - 'scripts/unity-compat-samples/gallery/**' + - 'maven/cil-translator/**' + - 'maven/unity-compat/**' + - 'maven/build-engine/src/main/java/com/codename1/maven/Unity*' + - 'maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*' + - 'scripts/unity-gallery-prebuilt.sh' + - '.github/workflows/unity-gallery-prebuilt.yml' + - '.github/actions/unity-gallery-prebuilt/**' - '.github/workflows/scripts-android.yml' - 'scripts/setup-workspace.sh' - 'scripts/ci/retry.sh' @@ -123,8 +143,15 @@ concurrency: cancel-in-progress: true jobs: + # The application has a Unity project in it. One Linux job compiles it and + # hands the result to the jobs below as an artifact; none of them installs + # the .NET SDK or builds the Unity modules. + unity-gallery: + uses: ./.github/workflows/unity-gallery-prebuilt.yml + build-android: name: Build Android ${{ matrix.name }} + needs: unity-gallery strategy: fail-fast: false matrix: @@ -272,6 +299,11 @@ jobs: # tried and is wrong here, because -Pcompile-android reads its jars # from maven/target/cn1-binaries, which this job never stages. python3 scripts/check-android-api-removals.py --require-all + # After everything that fills the local Maven repository and before the + # application is built; this job has no .NET SDK (see the action). + - name: Unpack the Unity gallery prebuilt + uses: ./.github/actions/unity-gallery-prebuilt + - name: Build Hello Codename One Android app id: build-android-app run: | diff --git a/.github/workflows/scripts-ios-native.yml b/.github/workflows/scripts-ios-native.yml index 37960a445f9..80910dcf0bc 100644 --- a/.github/workflows/scripts-ios-native.yml +++ b/.github/workflows/scripts-ios-native.yml @@ -8,6 +8,11 @@ on: port_prepared: type: boolean default: false + # True when the caller already ran unity-gallery-prebuilt.yml in this run: + # its artifact has a fixed name and cannot be uploaded twice. + unity_prepared: + type: boolean + default: false concurrency: # Only the latest revision needs validation. PR numbers keep forks isolated; @@ -16,13 +21,19 @@ concurrency: cancel-in-progress: true jobs: + # The application has a Unity project in it. One Linux job compiles it and + # hands the result to the job below as an artifact, unless the caller did. + unity-gallery: + if: ${{ !inputs.unity_prepared }} + uses: ./.github/workflows/unity-gallery-prebuilt.yml + build-port: if: ${{ !inputs.port_prepared }} uses: ./.github/workflows/_build-ios-port.yml native-ios: - if: ${{ !cancelled() && (inputs.port_prepared || needs.build-port.result == 'success') }} - needs: build-port + if: ${{ !cancelled() && (inputs.port_prepared || needs.build-port.result == 'success') && (inputs.unity_prepared || needs.unity-gallery.result == 'success') }} + needs: [build-port, unity-gallery] permissions: contents: read runs-on: macos-15 @@ -117,6 +128,11 @@ jobs: # build-port handed off (not LRU-evicted, survives --failed re-runs). tar -C / -xf "$RUNNER_TEMP/cn1-built-bundle.tar" + # After everything that fills the local Maven repository and before the + # application is built; this job has no .NET SDK (see the action). + - name: Unpack the Unity gallery prebuilt + uses: ./.github/actions/unity-gallery-prebuilt + - name: Build sample iOS app id: build_ios_app run: ./scripts/build-ios-app.sh -q -DskipTests diff --git a/.github/workflows/scripts-ios.yml b/.github/workflows/scripts-ios.yml index f55f56bd844..30cdfb9833a 100644 --- a/.github/workflows/scripts-ios.yml +++ b/.github/workflows/scripts-ios.yml @@ -12,6 +12,16 @@ on: - cron: 15 1 * * * pull_request: paths: + # The application these jobs build compiles a Unity project, from an + # archive that unity-gallery-prebuilt.yml makes out of these. + - scripts/unity-compat-samples/gallery/** + - maven/cil-translator/** + - maven/unity-compat/** + - maven/build-engine/src/main/java/com/codename1/maven/Unity* + - maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity* + - scripts/unity-gallery-prebuilt.sh + - .github/workflows/unity-gallery-prebuilt.yml + - .github/actions/unity-gallery-prebuilt/** - .github/workflows/scripts-ios.yml - .github/workflows/_build-ios-port.yml - scripts/setup-workspace.sh @@ -68,6 +78,16 @@ on: branches: - master paths: + # The application these jobs build compiles a Unity project, from an + # archive that unity-gallery-prebuilt.yml makes out of these. + - scripts/unity-compat-samples/gallery/** + - maven/cil-translator/** + - maven/unity-compat/** + - maven/build-engine/src/main/java/com/codename1/maven/Unity* + - maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity* + - scripts/unity-gallery-prebuilt.sh + - .github/workflows/unity-gallery-prebuilt.yml + - .github/actions/unity-gallery-prebuilt/** - .github/workflows/scripts-ios.yml - .github/workflows/_build-ios-port.yml - scripts/setup-workspace.sh @@ -128,6 +148,14 @@ concurrency: cancel-in-progress: true jobs: + # The application has a Unity project in it. One Linux job compiles it and + # hands the result to the jobs below as an artifact; none of them installs + # the .NET SDK or builds the Unity modules. + unity-gallery: + needs: select + if: needs.select.outputs.any == 'true' + uses: ./.github/workflows/unity-gallery-prebuilt.yml + select: runs-on: ubuntu-24.04 outputs: @@ -147,13 +175,16 @@ jobs: needs: select if: needs.select.outputs.any == 'true' uses: ./.github/workflows/_build-ios-port.yml + with: + # One test server for every suite job of the run; see the step there. + test_server: true build-ios-metal: if: ${{ !inputs.watch_only && needs.select.outputs.ios == 'true' }} # The iOS renderer job. Metal is the only iOS rendering backend -- the # OpenGL ES 2 pipeline and the job that exercised it are gone -- so this # runs on every pull request rather than only on the nightly schedule. - needs: [select, build-port] + needs: [select, build-port, unity-gallery] permissions: contents: read pull-requests: write @@ -262,6 +293,17 @@ jobs: # ~/.m2/com/codenameone + Themes + Ports/iOSPort/nativeSources to # their original absolute paths. tar -C / -xf "$RUNNER_TEMP/cn1-built-bundle.tar" + # The test server build-port built for the whole run, when it did. + # Without it the suite script builds its own, as it always has. + if [ -f "$RUNNER_TEMP/cn1ss-server.tar" ]; then + tar -C / -xf "$RUNNER_TEMP/cn1ss-server.tar" + fi + if [ -d "$RUNNER_TEMP/cn1ss-server/classes" ]; then + echo "CN1SS_SERVER_DIST=$RUNNER_TEMP/cn1ss-server" >> "$GITHUB_ENV" + echo "Using the prebuilt test server in $RUNNER_TEMP/cn1ss-server" + else + echo "No prebuilt test server in the bundle; the suite builds its own." + fi - name: Install Metal Toolchain # Xcode 26+ requires the Metal Toolchain component to be downloaded @@ -281,6 +323,11 @@ jobs: "$XCODEBUILD" -downloadComponent MetalToolchain timeout-minutes: 10 + # After everything that fills the local Maven repository and before the + # application is built; this job has no .NET SDK (see the action). + - name: Unpack the Unity gallery prebuilt + uses: ./.github/actions/unity-gallery-prebuilt + - name: Build sample iOS app and compile workspace (Metal) id: build-ios-app run: ./scripts/build-ios-app.sh -q -DskipTests @@ -540,7 +587,7 @@ jobs: # `xcode-27` is a PUBLIC PREVIEW image (actions/runner-images#14404) whose # capacity is still being balanced, so read a queue here as the image rather # than as a fault in the change under test. - needs: [select, build-port] + needs: [select, build-port, unity-gallery] permissions: contents: read pull-requests: write @@ -660,6 +707,17 @@ jobs: # ~/.m2/com/codenameone + Themes + Ports/iOSPort/nativeSources to # their original absolute paths. tar -C / -xf "$RUNNER_TEMP/cn1-built-bundle.tar" + # The test server build-port built for the whole run, when it did. + # Without it the suite script builds its own, as it always has. + if [ -f "$RUNNER_TEMP/cn1ss-server.tar" ]; then + tar -C / -xf "$RUNNER_TEMP/cn1ss-server.tar" + fi + if [ -d "$RUNNER_TEMP/cn1ss-server/classes" ]; then + echo "CN1SS_SERVER_DIST=$RUNNER_TEMP/cn1ss-server" >> "$GITHUB_ENV" + echo "Using the prebuilt test server in $RUNNER_TEMP/cn1ss-server" + else + echo "No prebuilt test server in the bundle; the suite builds its own." + fi - name: Install Metal Toolchain # Xcode 26+ requires the Metal Toolchain component to be downloaded @@ -679,6 +737,11 @@ jobs: "$XCODEBUILD" -downloadComponent MetalToolchain timeout-minutes: 10 + # After everything that fills the local Maven repository and before the + # application is built; this job has no .NET SDK (see the action). + - name: Unpack the Unity gallery prebuilt + uses: ./.github/actions/unity-gallery-prebuilt + - name: Build sample iOS app and compile workspace (Metal) id: build-ios-app run: ./scripts/build-ios-app.sh -q -DskipTests @@ -763,6 +826,15 @@ jobs: # The device TYPE is available even though no instance is, so create one. # Same model as the iOS 26 leg on purpose: the two sets then differ by # runtime and theme, which is what they are for, rather than by hardware. + # + # Booted, proven, and shut down again: this step is here to create the + # device and to find out on a deadline that its runtime boots, not to + # keep it running. run-ios-ui-tests.sh boots the device itself (simctl + # boot, then bootstatus -b) before it compiles and again before it + # installs. See the script for what the first boot was costing the + # step after it. + env: + CN1_SIM_SHUTDOWN_AFTER_BOOT: '1' run: ./scripts/ci/boot-ios-simulator.sh 'iOS-27[0-9-]*' 'iPhone 16' iPhone16-metal27 - name: Run iOS UI screenshot tests (Metal) env: @@ -946,7 +1018,7 @@ jobs: # WS sink the iOS jobs use, and compares against scripts/ios/screenshots-watch. # Isolated in its own job (like build-ios-metal) so the watch result is a # distinct check and a regression there doesn't mask the iOS path. - needs: [select, build-port] + needs: [select, build-port, unity-gallery] permissions: contents: read pull-requests: write @@ -1031,6 +1103,22 @@ jobs: # Replaces the evictable cn1-built cache restore with the artifact # build-port handed off (not LRU-evicted, survives --failed re-runs). tar -C / -xf "$RUNNER_TEMP/cn1-built-bundle.tar" + # The test server build-port built for the whole run, when it did. + # Without it the suite script builds its own, as it always has. + if [ -f "$RUNNER_TEMP/cn1ss-server.tar" ]; then + tar -C / -xf "$RUNNER_TEMP/cn1ss-server.tar" + fi + if [ -d "$RUNNER_TEMP/cn1ss-server/classes" ]; then + echo "CN1SS_SERVER_DIST=$RUNNER_TEMP/cn1ss-server" >> "$GITHUB_ENV" + echo "Using the prebuilt test server in $RUNNER_TEMP/cn1ss-server" + else + echo "No prebuilt test server in the bundle; the suite builds its own." + fi + + # After everything that fills the local Maven repository and before the + # application is built; this job has no .NET SDK (see the action). + - name: Unpack the Unity gallery prebuilt + uses: ./.github/actions/unity-gallery-prebuilt - name: Build sample iOS app (generates the watch target) id: build-ios-app @@ -1106,7 +1194,7 @@ jobs: # BLOCKING (a hard golden gate, like build-ios-watch): the tvOS slice # compiles end-to-end and the golden set is seeded from a CI capture (see # Ports/iOSPort/nativeSources/TVOS_PORT.md). A mismatch fails the job. - needs: [select, build-port] + needs: [select, build-port, unity-gallery] permissions: contents: read pull-requests: write @@ -1187,6 +1275,22 @@ jobs: # Replaces the evictable cn1-built cache restore with the artifact # build-port handed off (not LRU-evicted, survives --failed re-runs). tar -C / -xf "$RUNNER_TEMP/cn1-built-bundle.tar" + # The test server build-port built for the whole run, when it did. + # Without it the suite script builds its own, as it always has. + if [ -f "$RUNNER_TEMP/cn1ss-server.tar" ]; then + tar -C / -xf "$RUNNER_TEMP/cn1ss-server.tar" + fi + if [ -d "$RUNNER_TEMP/cn1ss-server/classes" ]; then + echo "CN1SS_SERVER_DIST=$RUNNER_TEMP/cn1ss-server" >> "$GITHUB_ENV" + echo "Using the prebuilt test server in $RUNNER_TEMP/cn1ss-server" + else + echo "No prebuilt test server in the bundle; the suite builds its own." + fi + + # After everything that fills the local Maven repository and before the + # application is built; this job has no .NET SDK (see the action). + - name: Unpack the Unity gallery prebuilt + uses: ./.github/actions/unity-gallery-prebuilt - name: Build sample iOS app (generates the tvOS target) id: build-ios-app @@ -1270,19 +1374,21 @@ jobs: retention-days: 14 native: - needs: [select, build-port] + needs: [select, build-port, unity-gallery] if: ${{ !inputs.watch_only && needs.select.outputs.native == 'true' }} uses: ./.github/workflows/scripts-ios-native.yml with: port_prepared: true + unity_prepared: true secrets: inherit packaging: - needs: [select, build-port] + needs: [select, build-port, unity-gallery] if: ${{ !inputs.watch_only && needs.select.outputs.packaging == 'true' }} uses: ./.github/workflows/ios-packaging.yml with: port_prepared: true + unity_prepared: true secrets: inherit catalyst: @@ -1290,9 +1396,10 @@ jobs: contents: read pull-requests: write issues: write - needs: [select, build-port] + needs: [select, build-port, unity-gallery] if: ${{ !inputs.watch_only && needs.select.outputs.catalyst == 'true' }} uses: ./.github/workflows/scripts-mac-catalyst.yml with: port_prepared: true + unity_prepared: true secrets: inherit diff --git a/.github/workflows/scripts-javascript.yml b/.github/workflows/scripts-javascript.yml index 5cfb2a650ea..571c51dd780 100644 --- a/.github/workflows/scripts-javascript.yml +++ b/.github/workflows/scripts-javascript.yml @@ -15,6 +15,16 @@ on: - cron: '35 1 * * *' pull_request: paths: + # The application these jobs build compiles a Unity project, from an + # archive that unity-gallery-prebuilt.yml makes out of these. + - 'scripts/unity-compat-samples/gallery/**' + - 'maven/cil-translator/**' + - 'maven/unity-compat/**' + - 'maven/build-engine/src/main/java/com/codename1/maven/Unity*' + - 'maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*' + - 'scripts/unity-gallery-prebuilt.sh' + - '.github/workflows/unity-gallery-prebuilt.yml' + - '.github/actions/unity-gallery-prebuilt/**' - 'scripts/ci/retry.sh' - '.github/workflows/scripts-javascript.yml' - 'scripts/run-javascript-browser-tests.sh' @@ -61,6 +71,16 @@ on: push: branches: [ master ] paths: + # The application these jobs build compiles a Unity project, from an + # archive that unity-gallery-prebuilt.yml makes out of these. + - 'scripts/unity-compat-samples/gallery/**' + - 'maven/cil-translator/**' + - 'maven/unity-compat/**' + - 'maven/build-engine/src/main/java/com/codename1/maven/Unity*' + - 'maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*' + - 'scripts/unity-gallery-prebuilt.sh' + - '.github/workflows/unity-gallery-prebuilt.yml' + - '.github/actions/unity-gallery-prebuilt/**' - 'scripts/ci/retry.sh' - '.github/workflows/scripts-javascript.yml' - 'scripts/run-javascript-browser-tests.sh' @@ -110,7 +130,15 @@ concurrency: cancel-in-progress: true jobs: + # The application has a Unity project in it. One Linux job compiles it and + # hands the result to the jobs below as an artifact; none of them installs + # the .NET SDK or builds the Unity modules. + unity-gallery: + if: ${{ !inputs.port_status_browser_evidence }} + uses: ./.github/workflows/unity-gallery-prebuilt.yml + javascript-screenshots: + needs: unity-gallery if: ${{ !inputs.port_status_browser_evidence }} permissions: contents: read @@ -296,6 +324,11 @@ jobs: - name: Setup workspace run: ./scripts/setup-workspace.sh -q -DskipTests + # After everything that fills the local Maven repository and before the + # application is built; this job has no .NET SDK (see the action). + - name: Unpack the Unity gallery prebuilt + uses: ./.github/actions/unity-gallery-prebuilt + - name: Build HelloCodenameOne JavaScript port bundle run: | # Use JAVA_HOME set by actions/setup-java for Java 17 diff --git a/.github/workflows/scripts-mac-catalyst.yml b/.github/workflows/scripts-mac-catalyst.yml index bec32802ab8..e456eb539d9 100644 --- a/.github/workflows/scripts-mac-catalyst.yml +++ b/.github/workflows/scripts-mac-catalyst.yml @@ -22,6 +22,11 @@ on: port_prepared: type: boolean default: false + # True when the caller already ran unity-gallery-prebuilt.yml in this run: + # its artifact has a fixed name and cannot be uploaded twice. + unity_prepared: + type: boolean + default: false concurrency: # Only the latest revision needs validation. PR numbers keep forks isolated; @@ -30,16 +35,24 @@ concurrency: cancel-in-progress: true jobs: + # The application has a Unity project in it. One Linux job compiles it and + # hands the result to the job below as an artifact, unless the caller did. + unity-gallery: + if: ${{ !inputs.unity_prepared }} + uses: ./.github/workflows/unity-gallery-prebuilt.yml + build-port: if: ${{ !inputs.port_prepared }} # Shared with scripts-ios.yml / scripts-ios-native.yml / ios-packaging.yml # via the cn1-built cache; first runner to land a fresh SHA populates it # and the others skip the rebuild. uses: ./.github/workflows/_build-ios-port.yml + with: + test_server: true build-mac-catalyst: - if: ${{ !cancelled() && (inputs.port_prepared || needs.build-port.result == 'success') }} - needs: build-port + if: ${{ !cancelled() && (inputs.port_prepared || needs.build-port.result == 'success') && (inputs.unity_prepared || needs.unity-gallery.result == 'success') }} + needs: [build-port, unity-gallery] permissions: contents: read pull-requests: write @@ -135,6 +148,17 @@ jobs: # Replaces the evictable cn1-built cache restore with the artifact # build-port handed off (not LRU-evicted, survives --failed re-runs). tar -C / -xf "$RUNNER_TEMP/cn1-built-bundle.tar" + # The test server build-port built for the whole run, when it did. + # Without it the suite script builds its own, as it always has. + if [ -f "$RUNNER_TEMP/cn1ss-server.tar" ]; then + tar -C / -xf "$RUNNER_TEMP/cn1ss-server.tar" + fi + if [ -d "$RUNNER_TEMP/cn1ss-server/classes" ]; then + echo "CN1SS_SERVER_DIST=$RUNNER_TEMP/cn1ss-server" >> "$GITHUB_ENV" + echo "Using the prebuilt test server in $RUNNER_TEMP/cn1ss-server" + else + echo "No prebuilt test server in the bundle; the suite builds its own." + fi - name: Install Metal Toolchain # Xcode 26+ requires the Metal Toolchain component for .metal @@ -151,6 +175,11 @@ jobs: "$XCODEBUILD" -downloadComponent MetalToolchain timeout-minutes: 10 + # After everything that fills the local Maven repository and before the + # application is built; this job has no .NET SDK (see the action). + - name: Unpack the Unity gallery prebuilt + uses: ./.github/actions/unity-gallery-prebuilt + - name: Build sample Mac Catalyst app and compile workspace id: build-mac-catalyst-app run: ./scripts/build-mac-catalyst-app.sh -q -DskipTests diff --git a/.github/workflows/scripts-macos.yml b/.github/workflows/scripts-macos.yml index 79b25f02119..29a90a1e1f2 100644 --- a/.github/workflows/scripts-macos.yml +++ b/.github/workflows/scripts-macos.yml @@ -22,6 +22,16 @@ on: - cron: '25 1 * * *' pull_request: paths: + # The application these jobs build compiles a Unity project, from an + # archive that unity-gallery-prebuilt.yml makes out of these. + - 'scripts/unity-compat-samples/gallery/**' + - 'maven/cil-translator/**' + - 'maven/unity-compat/**' + - 'maven/build-engine/src/main/java/com/codename1/maven/Unity*' + - 'maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*' + - 'scripts/unity-gallery-prebuilt.sh' + - '.github/workflows/unity-gallery-prebuilt.yml' + - '.github/actions/unity-gallery-prebuilt/**' - '.github/workflows/scripts-macos.yml' - '.github/workflows/_build-mac-port.yml' - 'scripts/setup-workspace.sh' @@ -76,6 +86,16 @@ on: push: branches: [ master ] paths: + # The application these jobs build compiles a Unity project, from an + # archive that unity-gallery-prebuilt.yml makes out of these. + - 'scripts/unity-compat-samples/gallery/**' + - 'maven/cil-translator/**' + - 'maven/unity-compat/**' + - 'maven/build-engine/src/main/java/com/codename1/maven/Unity*' + - 'maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*' + - 'scripts/unity-gallery-prebuilt.sh' + - '.github/workflows/unity-gallery-prebuilt.yml' + - '.github/actions/unity-gallery-prebuilt/**' - '.github/workflows/scripts-macos.yml' - '.github/workflows/_build-mac-port.yml' - 'scripts/setup-workspace.sh' @@ -121,13 +141,19 @@ concurrency: cancel-in-progress: true jobs: + # The application has a Unity project in it. One Linux job compiles it and + # hands the result to the jobs below as an artifact; none of them installs + # the .NET SDK or builds the Unity modules. + unity-gallery: + uses: ./.github/workflows/unity-gallery-prebuilt.yml + build-port: # AppKit has its own source hash and producer; it does not invalidate # the iOS coordinator's bundle. uses: ./.github/workflows/_build-mac-port.yml build-macos: - needs: build-port + needs: [build-port, unity-gallery] permissions: contents: read pull-requests: write @@ -234,6 +260,11 @@ jobs: "$XCODEBUILD" -downloadComponent MetalToolchain timeout-minutes: 10 + # After everything that fills the local Maven repository and before the + # application is built; this job has no .NET SDK (see the action). + - name: Unpack the Unity gallery prebuilt + uses: ./.github/actions/unity-gallery-prebuilt + - name: Build sample macOS app and compile workspace id: build-macos-app run: ./scripts/build-macos-app.sh -q -DskipTests diff --git a/.github/workflows/unity-compat.yml b/.github/workflows/unity-compat.yml new file mode 100644 index 00000000000..58850cde64a --- /dev/null +++ b/.github/workflows/unity-compat.yml @@ -0,0 +1,369 @@ +name: Unity compatibility + +# Unity 2D projects built as Codename One applications: the CIL translator and +# scene compiler (maven/cil-translator) and the runtime translated scripts run +# against (maven/unity-compat). Both live behind the unity-compat Maven profile, +# because building them needs the .NET SDK and the framework build must not. +# +# unit tests the metadata reader against .NET's own +# System.Reflection.Metadata, the console sample's +# output against `dotnet run`, and the 2D scene's +# trace; -Dunity.requireDotnet=true turns "no .NET +# SDK, skipped" into a failure +# unity-compat-test.sh every sample under scripts/unity-compat-samples +# run on the JVM, through ParparVM's C target as a +# native executable and through its JavaScript +# target under Node, all traces identical +# unity-compat-project-test.sh +# the build integration: a sample imported into an +# archetype application with cn1:import-unity-project, +# built by the application's own Maven build and +# staged for every target, then converted to +# Gradle and staged again: the same entries +# static analysis SpotBugs (zero findings), PMD (the forbidden +# list) and Checkstyle (zero errors) over both +# modules, enforced by the script PR CI uses for +# the rest of the tree, and the ClassCastException +# gate over the runtime, which ParparVM translates +# unity-gallery-consumer the archive unity-gallery-prebuilt.yml makes for +# the screenshot workflows, unpacked into a job +# that built neither Unity module and has no SDK +# it may use: the application's common module has +# to build from it, and a stale archive has to be +# refused by name +# +# The static analysis gates run HERE and not in pr.yml's Java 8 leg, where the +# other modules are judged. Both modules build only with the .NET SDK, and +# putting them in that leg would make every pull request in the repository +# install .NET and build C# to analyse code it did not touch. Nothing is lost +# by keeping them apart: the `paths` below name both modules, the rulesets and +# the scripts that do the judging, so no change to any of them can merge +# without this workflow running, and the report script fails on a module that +# produced no report rather than reading the absence as clean. +on: + pull_request: + branches: [ master ] + paths: + - 'maven/cil-translator/**' + - 'maven/unity-compat/**' + - 'maven/pom.xml' + - 'maven/integration-tests/unity-compat-test.sh' + - 'maven/integration-tests/unity-compat-project-test.sh' + - 'maven/build-engine/src/main/java/com/codename1/maven/Unity*' + - 'maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*' + - 'maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/AppSupport.java' + - 'maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/tasks/CompileUnityTask.java' + - 'maven/build-engine/src/main/java/com/codename1/maven/GradleConversion.java' + - 'maven/cn1app-archetype/**' + - 'maven/integration-tests/inc/**' + - 'scripts/unity-compat-samples/**' + - 'CodenameOne/src/com/codename1/gaming/**' + # The platform the offscreen check of the game view runs on. + - 'maven/core-unittests/src/test/java/com/codename1/testing/**' + - 'vm/ByteCodeTranslator/**' + - 'vm/JavaAPI/**' + - 'vm/selfhost/js/**' + # What judges the two modules: a change that weakens a gate has to run it. + - '.github/scripts/generate-quality-report.py' + - 'maven/core-unittests/pmd.xml' + - 'maven/core-unittests/checkstyle.xml' + - 'scripts/check-cast-semantics.sh' + - 'scripts/cast-semantics-baseline.txt' + # The archive the screenshot workflows build the application from, + # and the one job here that proves a job without the SDK can use it. + - 'scripts/unity-gallery-prebuilt.sh' + - '.github/workflows/unity-gallery-prebuilt.yml' + - '.github/actions/unity-gallery-prebuilt/**' + - 'scripts/hellocodenameone/pom.xml' + - 'scripts/hellocodenameone/common/pom.xml' + - '.github/workflows/unity-compat.yml' + push: + branches: [ master ] + paths: + - 'maven/cil-translator/**' + - 'maven/unity-compat/**' + - 'maven/pom.xml' + - 'maven/integration-tests/unity-compat-test.sh' + - 'maven/integration-tests/unity-compat-project-test.sh' + - 'maven/build-engine/src/main/java/com/codename1/maven/Unity*' + - 'maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*' + - 'maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/AppSupport.java' + - 'maven/codenameone-gradle-plugin/src/main/java/com/codename1/gradle/tasks/CompileUnityTask.java' + - 'maven/build-engine/src/main/java/com/codename1/maven/GradleConversion.java' + - 'maven/cn1app-archetype/**' + - 'maven/integration-tests/inc/**' + - 'scripts/unity-compat-samples/**' + - 'CodenameOne/src/com/codename1/gaming/**' + # The platform the offscreen check of the game view runs on. + - 'maven/core-unittests/src/test/java/com/codename1/testing/**' + - 'vm/ByteCodeTranslator/**' + - 'vm/JavaAPI/**' + - 'vm/selfhost/js/**' + # What judges the two modules: a change that weakens a gate has to run it. + - '.github/scripts/generate-quality-report.py' + - 'maven/core-unittests/pmd.xml' + - 'maven/core-unittests/checkstyle.xml' + - 'scripts/check-cast-semantics.sh' + - 'scripts/cast-semantics-baseline.txt' + # The archive the screenshot workflows build the application from, + # and the one job here that proves a job without the SDK can use it. + - 'scripts/unity-gallery-prebuilt.sh' + - '.github/workflows/unity-gallery-prebuilt.yml' + - '.github/actions/unity-gallery-prebuilt/**' + - 'scripts/hellocodenameone/pom.xml' + - 'scripts/hellocodenameone/common/pom.xml' + - '.github/workflows/unity-compat.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + unity-compat: + runs-on: ubuntu-24.04 + timeout-minutes: 60 + env: + DOTNET_CLI_TELEMETRY_OPTOUT: '1' + DOTNET_NOLOGO: '1' + steps: + - uses: actions/checkout@v6 + - name: Set up JDK 8 + uses: actions/setup-java@v5 + with: + java-version: '8' + distribution: 'temurin' + cache: 'maven' + # The scripts and the runtime's C# target netstandard2.1, which any SDK + # builds, but the console sample is also built for net10.0 (that build is + # the `dotnet run` oracle) and the metadata dumper is a net10.0 program. + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + - name: Set up Node + uses: actions/setup-node@v6 + with: + node-version: '20' + # xvfb as in the other Maven builds; clang for the native executables. + - name: Install xvfb and clang + run: bash scripts/ci/apt-get-install.sh xvfb clang + - name: Build the core, the translator and the runtime + env: + RETRY_ONLY_MATCHING: transient + RETRY_ATTEMPTS: '5' + RETRY_DELAY_SECONDS: '60' + run: | + cd maven + bash ../scripts/ci/retry.sh xvfb-run -a mvn -B install -Plocal-dev-javase -Dunity-compat -DskipTests -Darchetype.test.skip=true -Dmaven.javadoc.skip=true -pl cil-translator,unity-compat,javase -am + # The samples are also run on a real game view with no window + # (player/offscreen-check.sh). That needs the JavaSE port's jar, built + # above for its software rasteriser and fonts, and the core unit tests' + # implementation of the platform, which is only compiled here: the + # module is in the `unittests` profile and `-am` above never reaches it. + - name: Compile the core unit tests' platform implementation + env: + RETRY_ONLY_MATCHING: transient + RETRY_ATTEMPTS: '5' + RETRY_DELAY_SECONDS: '60' + run: | + cd maven + bash ../scripts/ci/retry.sh mvn -B test-compile -DunitTests -DskipTests -Dmaven.javadoc.skip=true -pl core-unittests -am + - name: Unit tests + working-directory: maven + run: xvfb-run -a mvn -B -Plocal-dev-javase -Dunity-compat -Dunity.requireDotnet=true -pl cil-translator,unity-compat test + # `verify` writes the three reports of each module and fails on its own + # for SpotBugs and Checkstyle. The reports are deleted first so that the + # step after this one reads this run's answer and never a cached one. + - name: Static analysis of the translator and the runtime + working-directory: maven + run: | + rm -f cil-translator/target/spotbugsXml.xml cil-translator/target/pmd.xml cil-translator/target/checkstyle-result.xml + rm -f unity-compat/target/spotbugsXml.xml unity-compat/target/pmd.xml unity-compat/target/checkstyle-result.xml + xvfb-run -a mvn -B -Plocal-dev-javase -Dunity-compat -DskipTests -pl cil-translator,unity-compat verify + # The same script, with the same forbidden PMD list, that judges the rest + # of the tree in pr.yml. PMD only writes a report, so this step is what + # enforces it; every module named must have produced each report. + - name: Enforce the quality gates + if: ${{ !cancelled() }} + env: + QUALITY_REPORT_TARGET_DIRS: maven/cil-translator/target:maven/unity-compat/target + QUALITY_REPORT_REQUIRED_SPOTBUGS: cil-translator:unity-compat + QUALITY_REPORT_REQUIRED_PMD: cil-translator:unity-compat + QUALITY_REPORT_REQUIRED_CHECKSTYLE: cil-translator:unity-compat + QUALITY_REPORT_TITLE: Unity compatibility quality report + run: python3 .github/scripts/generate-quality-report.py + - name: Build ParparVM's class library and translator + run: | + bash scripts/ci/retry.sh mvn -B -f vm/JavaAPI/pom.xml package -DskipTests + bash scripts/ci/retry.sh mvn -B -f vm/ByteCodeTranslator/pom.xml package -DskipTests + # The runtime and the classes translated from C# are translated again by + # ParparVM, whose CHECKCAST does not throw. Named explicitly: the default + # roots belong to pr.yml, which builds them. + - name: No reliance on ClassCastException in the runtime + run: | + test -d maven/unity-compat/target/classes/UnityEngine + scripts/check-cast-semantics.sh maven/unity-compat/target/classes + # The offscreen check runs the JavaSE port's classes, which need JDK 11 + # or newer; the job's own JDK is 8. The runner image carries a 17. + - name: Samples on the JVM, native and JavaScript + working-directory: maven/integration-tests + run: | + test -x "${JAVA_HOME_17_X64}/bin/javac" + UNITY_PLAYER_JAVA_HOME="${JAVA_HOME_17_X64}" bash unity-compat-test.sh --parparvm + - name: Upload the samples' logs and traces + if: failure() + uses: actions/upload-artifact@v7 + with: + name: unity-compat-logs + path: | + maven/integration-tests/build/unity-compat/**/*.log + maven/integration-tests/build/unity-compat/**/*.txt + maven/integration-tests/build/unity-compat/**/*.err + maven/integration-tests/build/unity-compat/**/*.diff + maven/cil-translator/target/unity-tests/**/*.txt + maven/cil-translator/target/surefire-reports/** + if-no-files-found: ignore + + # The build integration needs the whole reactor -- the Maven plugin and the + # archetype, not just the translator and the runtime -- so it is a job of its + # own rather than a longer build in front of the samples. + unity-compat-project: + runs-on: ubuntu-24.04 + timeout-minutes: 60 + env: + DOTNET_CLI_TELEMETRY_OPTOUT: '1' + DOTNET_NOLOGO: '1' + steps: + - uses: actions/checkout@v6 + # Installed FIRST so the JDK 8 below is the one JAVA_HOME names. + - name: Set up JDK 17 for the application build + uses: actions/setup-java@v5 + with: + java-version: '17' + distribution: 'temurin' + - run: echo "GRADLE_JAVA_HOME=$JAVA_HOME" >> $GITHUB_ENV + - name: Set up JDK 8 + uses: actions/setup-java@v5 + with: + java-version: '8' + distribution: 'temurin' + cache: 'maven' + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + # xvfb for the CSS compiler, which opens an AWT frame. + - name: Install xvfb + run: bash scripts/ci/apt-get-install.sh xvfb + - name: Build Codename One Maven artifacts with the Unity profile + env: + RETRY_ONLY_MATCHING: transient + RETRY_ATTEMPTS: '5' + RETRY_DELAY_SECONDS: '60' + run: | + cd maven + bash ../scripts/ci/retry.sh xvfb-run -a mvn -B install -Plocal-dev-javase -Dunity-compat -DskipTests -Darchetype.test.skip=true -Dmaven.javadoc.skip=true + - name: Unit tests of the builder and the goals + working-directory: maven + run: xvfb-run -a mvn -B -Plocal-dev-javase -pl build-engine,codenameone-maven-plugin test -Dtest='Unity*Test,CompileUnityMojoTest' -Dsurefire.failIfNoSpecifiedTests=false + - name: Refresh archetype catalog + run: | + cd maven + xvfb-run -a mvn archetype:update-local-catalog -Plocal-dev-javase + xvfb-run -a mvn archetype:crawl -Plocal-dev-javase + - name: A Unity project imported and built as an application + working-directory: maven/integration-tests + env: + CN1_GRADLE_REQUIRED: '1' + run: xvfb-run -a bash unity-compat-project-test.sh + - name: Upload the build logs + if: failure() + uses: actions/upload-artifact@v7 + with: + name: unity-compat-project-logs + path: | + maven/integration-tests/build/unity-compat-project/*.log + maven/integration-tests/build/unity-compat-project/*.txt + maven/integration-tests/build/unity-compat-project/*.diff + maven/integration-tests/build/unity-compat-project/*.err + maven/integration-tests/build/unity-compat-project/app/common/target/unity/*.log + if-no-files-found: ignore + + # The screenshot workflows build scripts/hellocodenameone, which has a Unity + # project in it, on runners with no .NET SDK: one Linux job compiles the + # project and they unpack what it made. This is that job, and below it the + # only test of the handoff that does not cost a Mac. + unity-gallery: + uses: ./.github/workflows/unity-gallery-prebuilt.yml + + unity-gallery-consumer: + needs: unity-gallery + runs-on: ubuntu-24.04 + timeout-minutes: 45 + env: + # An SDK setting that names nothing is an error, never a reason to look + # elsewhere, so with this no step below can compile C# -- whatever the + # runner image has on its PATH. + NO_DOTNET: -Dcn1.unity.dotnet=/no/such/dotnet + steps: + - uses: actions/checkout@v6 + # Installed FIRST so the JDK 8 below is the one JAVA_HOME names. + - name: Set up JDK 17 for the application build + uses: actions/setup-java@v5 + with: + java-version: '17' + distribution: 'temurin' + - run: echo "JDK_17_HOME=$JAVA_HOME" >> $GITHUB_ENV + - name: Set up JDK 8 + uses: actions/setup-java@v5 + with: + java-version: '8' + distribution: 'temurin' + cache: 'maven' + # xvfb for the CSS compiler, which opens an AWT frame. + - name: Install xvfb + run: bash scripts/ci/apt-get-install.sh xvfb + # What a screenshot job builds of the framework: no -Dunity-compat. The + # two artifacts are removed first because the Maven cache above is shared + # with the jobs of this workflow that do build them. + - name: Build the framework without the Unity modules + run: | + rm -rf ~/.m2/repository/com/codenameone/codenameone-unity-compat ~/.m2/repository/com/codenameone/codenameone-cil-translator + cd maven + bash ../scripts/ci/retry.sh xvfb-run -a mvn -B install -Plocal-dev-javase -DskipTests -Darchetype.test.skip=true -Dmaven.javadoc.skip=true -pl codenameone-maven-plugin,cn1-ads-mock,android-compat -am + test ! -e ~/.m2/repository/com/codenameone/codenameone-unity-compat + - name: Unpack the Unity gallery prebuilt + uses: ./.github/actions/unity-gallery-prebuilt + - name: Build the application's common module from the archive + run: | + set -euo pipefail + export JAVA_HOME="$JDK_17_HOME" + export PATH="$JAVA_HOME/bin:$PATH" + mkdir -p "$HOME/.codenameone" + [ -f "$HOME/.codenameone/guibuilder.jar" ] || : > "$HOME/.codenameone/guibuilder.jar" + common=scripts/hellocodenameone/common + cp "$common/target/unity/state.txt" "$RUNNER_TEMP/unity-state.txt" + bash scripts/ci/retry.sh xvfb-run -a mvn -B -f scripts/hellocodenameone/pom.xml -pl common -am install -DskipTests -Dmaven.javadoc.skip=true "$NO_DOTNET" + cmp "$RUNNER_TEMP/unity-state.txt" "$common/target/unity/state.txt" + test ! -e "$common/target/unity/dotnet.log" + test -f "$common/target/classes/com/codename1/generated/unity/UnityAppImpl.class" + # The other half: output that does not belong to these sources and jars + # is not used, and the failure says that rather than only asking for an + # SDK. This is what a screenshot job prints when its archive is stale. + - name: A stale archive is refused by name + run: | + set -uo pipefail + export JAVA_HOME="$JDK_17_HOME" + export PATH="$JAVA_HOME/bin:$PATH" + echo stale > scripts/hellocodenameone/common/target/unity/state.txt + # The step runs under "bash -e": the build is meant to fail, so its + # status is taken with "||" or the step would stop before judging it. + status=0 + xvfb-run -a mvn -B -f scripts/hellocodenameone/pom.xml -pl common generate-sources -DskipTests "$NO_DOTNET" > "$RUNNER_TEMP/stale.log" 2>&1 || status=$? + tail -n 30 "$RUNNER_TEMP/stale.log" + if [ "$status" -eq 0 ]; then + echo "::error::A build with a stale state.txt and no .NET SDK succeeded." + exit 1 + fi + grep -q 'does not match these sources or jars' "$RUNNER_TEMP/stale.log" + grep -q 'unity-compat-samples/gallery' "$RUNNER_TEMP/stale.log" diff --git a/.github/workflows/unity-gallery-prebuilt.yml b/.github/workflows/unity-gallery-prebuilt.yml new file mode 100644 index 00000000000..b8d0c9f5365 --- /dev/null +++ b/.github/workflows/unity-gallery-prebuilt.yml @@ -0,0 +1,165 @@ +name: _Unity gallery prebuilt (reusable) + +# scripts/hellocodenameone, the application every screenshot workflow builds, +# contains a Unity project (scripts/unity-compat-samples/gallery). Compiling +# it takes the .NET SDK and the two Maven modules behind -Dunity-compat. This +# workflow is the ONE place that happens: it compiles the gallery on Linux, +# packs the result with scripts/unity-gallery-prebuilt.sh and uploads it as the +# artifact `unity-gallery-prebuilt`. Every job that builds the application +# `needs:` the job that calls this workflow and runs +# ./.github/actions/unity-gallery-prebuilt, which downloads and unpacks it. +# No macOS, Windows, Android or iOS job installs .NET or builds those modules. +# +# What the consumer gets, and why it is safe on another machine and another +# operating system, is described in scripts/unity-gallery-prebuilt.sh. +# +# Calling it: +# +# unity-gallery: +# uses: ./.github/workflows/unity-gallery-prebuilt.yml +# +# ONCE per workflow run. The artifact has a fixed name, and a second upload of +# it in the same run is refused, so a workflow that is itself called by one +# that already made the archive takes an input saying so and skips its own +# call (`unity_prepared`, beside `port_prepared`, in scripts-ios-native.yml, +# ios-packaging.yml and scripts-mac-catalyst.yml). +# +# The cache. A cold build is the framework up to the Maven plugin plus both +# Unity modules, several minutes; the archive that comes out of it depends only +# on the files hashed into the key below, so it is cached under that hash and a +# run that finds it does nothing but upload it again. Two things about that: +# +# - The key has to name everything that can change a byte of the archive. The +# consumer's check is by content, but of the archive's OWN jars: a stale +# archive is not refused downstream, it is used. So the key takes the whole +# core (the runtime is compiled against it, and javac copies constants and +# binds overloads at compile time), both Unity modules, the builder and the +# goal that drive the compile, the gallery, the application's poms and the +# scripts and workflow that make the archive. The .NET SDK is not in it: +# another SDK would write other bytes for the same sources, and either +# archive is right. +# - GitHub scopes a cache to the ref that saved it. A pull request's runs all +# have the same ref (refs/pull/N/merge), so the screenshot workflows of one +# pull request share one entry, and a run can also read what master saved. +# Workflows that start together on a push all miss together and each +# builds; the later ones find the entry. Another pull request never sees it. +# +# The artifact, not the cache, is the handoff within a run: a cache entry can +# be evicted between this job and a consumer that waited for a runner. + +on: + workflow_call: + +jobs: + build: + name: Unity gallery prebuilt + runs-on: ubuntu-24.04 + timeout-minutes: 45 + permissions: + contents: read + env: + DOTNET_CLI_TELEMETRY_OPTOUT: '1' + DOTNET_NOLOGO: '1' + PACK: ${{ github.workspace }}/artifacts/unity-gallery-prebuilt/unity-gallery-prebuilt.tgz + steps: + - uses: actions/checkout@v6 + + # Everything below this step but the upload is skipped on a hit. + - name: Cache the Unity gallery prebuilt + id: pack + uses: actions/cache@v5 + with: + path: artifacts/unity-gallery-prebuilt/unity-gallery-prebuilt.tgz + key: unity-gallery-prebuilt-v1-${{ hashFiles('maven/pom.xml', 'maven/cil-translator/**', 'maven/unity-compat/**', 'maven/build-engine/src/main/java/com/codename1/maven/Unity*', 'maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*', 'CodenameOne/src/**', 'Ports/CLDC11/src/**', 'scripts/unity-compat-samples/gallery/**', 'scripts/hellocodenameone/pom.xml', 'scripts/hellocodenameone/common/pom.xml', 'scripts/unity-gallery-prebuilt.sh', '.github/workflows/unity-gallery-prebuilt.yml') }} + + # Installed FIRST so the JDK 8 below is the one JAVA_HOME names. + - name: Set up JDK 17 for the application build + if: steps.pack.outputs.cache-hit != 'true' + uses: actions/setup-java@v5 + with: + java-version: '17' + distribution: 'temurin' + - if: steps.pack.outputs.cache-hit != 'true' + run: echo "JDK_17_HOME=$JAVA_HOME" >> $GITHUB_ENV + - name: Set up JDK 8 + if: steps.pack.outputs.cache-hit != 'true' + uses: actions/setup-java@v5 + with: + java-version: '8' + distribution: 'temurin' + cache: 'maven' + - name: Set up .NET + if: steps.pack.outputs.cache-hit != 'true' + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + # xvfb as in the other Maven builds of the framework. + - name: Install xvfb + if: steps.pack.outputs.cache-hit != 'true' + run: bash scripts/ci/apt-get-install.sh xvfb + + # The least of the framework that compiles the gallery: the Maven plugin + # (the compile-unity goal) and what the application's common module + # depends on, plus the two Unity modules. These steps only build, so a + # blanket retry cannot turn a flaky test into a pass. + - name: Build the plugin, the Unity modules and the application's dependencies + if: steps.pack.outputs.cache-hit != 'true' + run: | + cd maven + bash ../scripts/ci/retry.sh xvfb-run -a mvn -B install -Plocal-dev-javase -Dunity-compat -DskipTests -Darchetype.test.skip=true -Dmaven.javadoc.skip=true -pl codenameone-maven-plugin,cn1-ads-mock,android-compat,cil-translator,unity-compat -am + + # generate-sources is as far as the application has to go: compile-unity + # is bound to it, and nothing after it writes what the archive carries. + - name: Compile the gallery + if: steps.pack.outputs.cache-hit != 'true' + run: | + set -euo pipefail + export JAVA_HOME="$JDK_17_HOME" + export PATH="$JAVA_HOME/bin:$PATH" + mkdir -p "$HOME/.codenameone" + [ -f "$HOME/.codenameone/guibuilder.jar" ] || : > "$HOME/.codenameone/guibuilder.jar" + bash scripts/ci/retry.sh xvfb-run -a mvn -B -f scripts/hellocodenameone/pom.xml -pl common -am generate-sources -DskipTests + test -s scripts/hellocodenameone/common/target/unity/state.txt + + - name: Pack + if: steps.pack.outputs.cache-hit != 'true' + run: bash scripts/unity-gallery-prebuilt.sh pack "$PACK" + + # On a hit as well: an entry that was cut short is caught here, in the + # job that can say so, and not in a dozen consumers. + - name: Check the archive + run: | + set -euo pipefail + ls -la "$PACK" + version="$(sed -n 's:.*\(.*\).*:\1:p' scripts/hellocodenameone/pom.xml | head -n 1)" + info="$(tar -xzOf - pack-info.txt <"$PACK")" + echo "$info" + echo "$info" | grep -qx "cn1.version=$version" + # The listing goes to a variable first: grep -q stops reading at its + # first match, and tar, still writing, then fails the pipeline with + # "write error" under pipefail -- on some runs and not on others. + entries="$(tar -tzf - <"$PACK")" + grep -qx 'target/unity/state.txt' <<<"$entries" + if [ "${{ steps.pack.outputs.cache-hit }}" = "true" ]; then + echo "Unity gallery prebuilt: cache HIT; nothing was built." + else + echo "Unity gallery prebuilt: cache MISS; built fresh." + fi + + - name: Upload the Unity gallery prebuilt + uses: actions/upload-artifact@v7 + with: + name: unity-gallery-prebuilt + path: ${{ env.PACK }} + retention-days: 1 + if-no-files-found: error + # A re-run of a failed attempt uploads under the same name. + overwrite: true + + - name: Upload the build logs + if: failure() + uses: actions/upload-artifact@v7 + with: + name: unity-gallery-prebuilt-logs + path: scripts/hellocodenameone/common/target/unity/*.log + if-no-files-found: ignore diff --git a/.github/workflows/windows-cross-build-run.yml b/.github/workflows/windows-cross-build-run.yml index dc3ad571469..627b5e404ef 100644 --- a/.github/workflows/windows-cross-build-run.yml +++ b/.github/workflows/windows-cross-build-run.yml @@ -26,6 +26,16 @@ on: - cron: '55 1 * * *' pull_request: paths: + # The application these jobs build compiles a Unity project, from an + # archive that unity-gallery-prebuilt.yml makes out of these. + - 'scripts/unity-compat-samples/gallery/**' + - 'maven/cil-translator/**' + - 'maven/unity-compat/**' + - 'maven/build-engine/src/main/java/com/codename1/maven/Unity*' + - 'maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*' + - 'scripts/unity-gallery-prebuilt.sh' + - '.github/workflows/unity-gallery-prebuilt.yml' + - '.github/actions/unity-gallery-prebuilt/**' - '.github/workflows/windows-cross-build-run.yml' - 'vm/**' - 'Ports/WindowsPort/**' @@ -44,6 +54,16 @@ on: push: branches: [ master, main ] paths: + # The application these jobs build compiles a Unity project, from an + # archive that unity-gallery-prebuilt.yml makes out of these. + - 'scripts/unity-compat-samples/gallery/**' + - 'maven/cil-translator/**' + - 'maven/unity-compat/**' + - 'maven/build-engine/src/main/java/com/codename1/maven/Unity*' + - 'maven/codenameone-maven-plugin/src/main/java/com/codename1/maven/*Unity*' + - 'scripts/unity-gallery-prebuilt.sh' + - '.github/workflows/unity-gallery-prebuilt.yml' + - '.github/actions/unity-gallery-prebuilt/**' - '.github/workflows/windows-cross-build-run.yml' - 'vm/**' - 'Ports/WindowsPort/**' @@ -59,7 +79,14 @@ concurrency: cancel-in-progress: true jobs: + # The application has a Unity project in it. One Linux job compiles it and + # hands the result to the jobs below as an artifact; none of them installs + # the .NET SDK or builds the Unity modules. + unity-gallery: + uses: ./.github/workflows/unity-gallery-prebuilt.yml + cross-build: + needs: unity-gallery name: cross-build suite exe (Linux, x64 + WebView2) runs-on: ubuntu-24.04 timeout-minutes: 60 @@ -187,6 +214,11 @@ jobs: test -f core/target/classes/com/codename1/ui/Form.class test -f windows/target/classes/com/codename1/impl/windows/WindowsImplementation.class + # After everything that fills the local Maven repository and before the + # application is built; this job has no .NET SDK (see the action). + - name: Unpack the Unity gallery prebuilt + uses: ./.github/actions/unity-gallery-prebuilt + # Build the hellocodenameone screenshot app into common/target/classes (the # suite the cross-build translates). Plain Maven build of the sample on JDK # 17 -- exactly what a fresh initializr project does; the plugin runs diff --git a/.gitignore b/.gitignore index a6304d15f17..296b1c98e39 100644 --- a/.gitignore +++ b/.gitignore @@ -173,6 +173,8 @@ docs/developer-guide/_generated-build-hints.adoc # and the symptom is "cannot find symbol" on a class this branch just added. # Build with -Dmaven.repo.local=$(pwd)/.m2-repo instead. /.m2-repo/ +/.dotnet/ +/.nuget-packages/ # Generated by .github/scripts/generate-quality-report.py, uploaded as a CI # artifact rather than committed. diff --git a/CLAUDE.md b/CLAUDE.md index fc26e59b1ab..23444108c8b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -115,7 +115,13 @@ build plugins share), `project-model` (the project-layout API), `android-res-compiler` and `android-compat` (the Android compatibility runtime), `codenameone-gradle-plugin` and `javac` (the in-tree Java compiler, `vm/JavaCompiler`), then enforces the result in -`.github/scripts/generate-quality-report.py`. +`.github/scripts/generate-quality-report.py`. `cil-translator` and +`unity-compat` (the Unity compatibility translator and runtime) are held to the +same script with SpotBugs, PMD and Checkstyle all three, but in +`.github/workflows/unity-compat.yml`: they build only with the .NET SDK, and +the Java 8 leg must not need one. That workflow's `paths` name both modules, +the core's `pmd.xml` and `checkstyle.xml` and the report script, so a change to +any of them cannot skip it. - **SpotBugs is a zero-findings gate.** *Any* finding of *any* pattern in *any* of those projects fails the build, and a project that produces no SpotBugs @@ -126,7 +132,8 @@ runtime), `codenameone-gradle-plugin` and `javac` (the in-tree Java compiler, `vm/ByteCodeTranslator/`, `maven/codenameone-maven-plugin/`, `maven/build-engine/`, `maven/project-model/`, `maven/android-res-compiler/`, `maven/android-compat/`, - `maven/codenameone-gradle-plugin/`, `maven/javac/`), scoped to the + `maven/codenameone-gradle-plugin/`, `maven/javac/`, + `maven/cil-translator/`, `maven/unity-compat/`), scoped to the class or method it applies to and with a comment explaining why. Keep the generated report at zero rather than tolerating known noise. - PMD and Checkstyle still gate on their own lists in the same script. @@ -141,6 +148,20 @@ mvn -B -DunitTests -DskipTests=true -pl core-unittests verify mvn -B -DskipTests=true -f ../vm/ByteCodeTranslator/pom.xml verify ``` +And the Unity compatibility modules, which need the .NET SDK. `verify` fails on +a SpotBugs finding or a Checkstyle error by itself; **PMD only writes +`target/pmd.xml`**, so the script is what judges it: + +```bash +cd maven && mvn -B -Dunity-compat -DskipTests=true -pl cil-translator,unity-compat clean verify +cd .. && QUALITY_REPORT_TARGET_DIRS=maven/cil-translator/target:maven/unity-compat/target \ + QUALITY_REPORT_REQUIRED_SPOTBUGS=cil-translator:unity-compat \ + QUALITY_REPORT_REQUIRED_PMD=cil-translator:unity-compat \ + QUALITY_REPORT_REQUIRED_CHECKSTYLE=cil-translator:unity-compat \ + python3 .github/scripts/generate-quality-report.py +scripts/check-cast-semantics.sh maven/unity-compat/target/classes +``` + **Run the `core-unittests` line too, and do not skip it because the first command "already covered" that module -- it does not.** `core-unittests` is declared inside the `unittests` profile, so `-am` never reaches it and its @@ -828,6 +849,253 @@ Traps that have already cost a fix: (`.github/workflows/android-compat.yml`); it fails if any shipped class still names `android/`. +### Unity compatibility: Unity 2D projects as Codename One apps + +A Unity project's `Assets` and `ProjectSettings` dropped into +`common/src/main/unity` (or imported with `cn1:import-unity-project`) builds +unmodified: + +- **`maven/cil-translator`** reads the assembly the C# compiler produced and + writes one class file per C# type (`com.codename1.cil`), and compiles every + `.unity` scene and `.prefab` to Java source, the generated + `com.codename1.generated.unity.UnityAppImpl` + (`com.codename1.unity.scenecompiler`). +- **`maven/unity-compat`** is the runtime the translated classes run against: + a .NET base library in Java, the `UnityEngine` value types (C#, translated at + build time) and the engine over `com.codename1.gaming`. Its `references` + classifier jar carries the three assemblies application scripts compile + against; `com.codename1.unitycompat.app.UnityApplication` is the lifecycle + class the imported main class extends. +- **The build step** (`build-engine/.../UnityProjectBuilder`, goal + `compile-unity` in `generate-sources`) runs `dotnet build` on a generated + project file, forks the translator and the scene compiler, and installs the + classes and images in the common module's classes. It has no compile + dependency on `cil-translator` -- the tool is resolved at the runtime's own + version and forked -- so `build-engine` builds without the profile below. +- **Gradle** runs the same builder as the `compileUnity` task + (`codenameone-gradle-plugin`, `AppSupport.registerUnity`), and adds the + runtime, the tool and the references itself for a project with + `src/main/unity`. The translated classes go to a directory of their own + (`build/cn1-unity/classes`, an output directory of the main source set and on + javac's classpath), never into `compileJava`'s destination: javac empties + that on a full recompile. The task reads the runtime jar from the + `compileClasspath` *configuration*; the source set's classpath contains the + task's own output, and reading that would make the task depend on itself. + +Both modules live behind the **`unity-compat` profile** (`-Dunity-compat`), +because building them needs the .NET SDK and the framework build must not. +An application needs the SDK only when it has Unity sources, and only for a +build that follows a change to them. + +Both are on the static analysis gates, at zero (see *Static Analysis Gates* for +where they run and how to reproduce them). Three things about the runtime's +exceptions, so that the next finding is fixed rather than added to them: + +- **C# member names are excused by package, nothing else is.** + `get_position`, `GetComponent` and `Item1` are what translated bytecode + calls, so SpotBugs' `NM_*` patterns are excluded for + `com.codename1.unitycompat.{system,unityengine}` and each class with such + members carries `@SuppressWarnings("PMD.MethodNamingConventions")`. A new + runtime class needs that annotation; a Java-named helper never does. +- **The translated classes are left out of SpotBugs by name** + (`UnityEngine.*`): they are generated bytecode in `target/classes` with no + Java source. PMD and Checkstyle read sources and never see them. A finding + there that points at wasted or wrong codegen is the translator's to fix. +- **Indexed loops stay indexed.** `// NOPMD ForLoopCanBeForeach` on a loop over + a list is deliberate: a for-each allocates an iterator, in code that runs + every frame, and several of those lists grow while they are walked. + +`scripts/check-cast-semantics.sh` covers `maven/unity-compat/target/classes` +(the runtime and the translated classes) whenever the module is built, and +`unity-compat.yml` names it explicitly. It is not one of the roots +`--require-all` demands, because PR CI's Java 8 leg does not build it. + +**A release publishes both**, in the core reactor pass and at the release's own +version, because the archetype and both build plugins resolve them at that +version. `release.yml` installs the SDK and passes `-Dunity-compat`; a release +build without the flag *succeeds* two modules short, so its "Confirm artifacts +on R2" step names both, and the `references` jar by file. `versions:set` reads +the modules of every profile, so `update-version.sh` bumps them unaided. + +Traps that have already cost a fix: + +- **A method returning a struct takes a hidden last parameter**, the object + the result is written into (`Names`, in `com.codename1.cil.translate`). Java + has no value types, `a + b` on two vectors is a call, and an object per call + measured an order of magnitude slower under ParparVM. A hand-written runtime + method that returns a value type follows the same convention, or the + translated call site passes one argument more than it declares; and the + reference it returns may be read but never kept. +- **A static field of a generic type is one per constructed type.** Erasure + makes every `Singleton` one class, so `GenericStatics` moves such fields + into a context object, `Type$$S`, with one subclass per constructed type + whose `` runs the type's `.cctor`; static methods take it as a hidden + last parameter, instances keep it in a field. What cannot be keyed fails the + build, never shares one field. +- **Float arithmetic has to round the way C# rounds.** Natively that needs + `-ffp-contract=off`, or clang fuses a multiply and an add and the last bit + differs from the JVM; on JavaScript every `float` result goes through + `Math.fround`, because a JavaScript number is a double. Without both, a + seeded game diverges between targets after a few hundred frames, which is + what the committed traces compare. +- **A script with no namespace lands in package `global`**, since a class in + Java's default package cannot be imported by the generated factory. +- **No reflection.** Scenes, prefabs and serialized fields compile to code + that calls constructors and assigns fields; `Invoke("name")`, `Instantiate` and + `AddComponent` go through methods the translator writes into each script + (`$invoke`, `$copyFrom`, `$new`). Nothing looks a name up at run time, + because ParparVM strips what nothing references. +- **A C# `T[,]` is one flat Java array**, never an array of arrays: an + `MdArray*` holder (`com.codename1.unitycompat.system`) keeps the lengths, + and `a[i, j]` is a static `get`/`set` that checks each dimension by itself -- + an index past the end of one row is otherwise a valid element of the next. +- **A helper method that takes an enum has the enum in its name** + (`Split$StringSplitOptions`, `Names.enumSuffix`). An enum is an `int` once + erased, so `Split(char[], int)` and `Split(char[], StringSplitOptions)` + would be one Java method. +- **`Resources.Load` is resolved at build time.** The scene compiler embeds + what is under any `Resources` folder and generates `loadResource`, a chain + of string comparisons; a kind it cannot load is a build note, and the call + answers null. +- **`UnityEngine/*.class` is translated twice** -- into the runtime jar and + again with the scripts. The builder skips every class the runtime jar + already has; a second copy in the application would shadow the runtime's. +- **A Unity message is sent only if three places agree.** Unity calls + `OnMouseDown` by its name; here the translator writes a `$onMouseDown` + override for a method in `Translator.MESSAGES`, `MonoBehaviour` declares + the empty hook, and `UnityRuntime` calls it. A documented message in none of + them is in `UNSENT_MESSAGES`, and a script declaring one gets a build + warning with its file and line. Dispatching a new one means all three and + taking it off that list -- otherwise it compiles and is never called. +- **Code outside the frame reaches a script through + `UnityRuntime.callInFrame`** (`UnityApplication.callInFrame`), never + directly: a port may run the frame off the EDT. The queue is an + `AtomicReference` to an immutable array, as the key events are. +- **Animation, tiles and particles are compiled, not loaded.** The scene + compiler turns every `.anim`, `.controller`, tile asset, tilemap and + particle module into code (`AnimationAssets`, `TilemapAssets`, + `ParticleAssets`); nothing parses YAML on the device. A property a curve + names that the runtime cannot write is a build note, never a silent skip. +- **TextMeshPro and Cinemachine are our own classes** + (`com.codename1.unitycompat.{tmpro,cinemachine}`), matched to a scene by the + package's script GUID (`SceneCompiler.PACKAGE_CLASSES`) because a package's + scripts are not under `Assets`. The packages' sources are never read. A new + runtime package has to be added in three places or a gate fails far from + the cause: the source roots the `cil-translator` tests compile, and both + package patterns in `maven/unity-compat/spotbugs-exclude.xml`. +- **A tilemap collider is outlines, not boxes.** Solid cells are merged and + traced into chains, so a body sliding along a floor meets no seam between + two tiles; `platformer2d`'s trace holds the speed of such a slide. Box2D + makes no contact between two chains, so a tilemap never collides with + another tilemap -- `World.popContact` used to throw on that pair. +- **ParparVM copied an array of primitive arrays as primitives.** `int[][]` + has a primitive element class and two dimensions; `System.arraycopy` and + `clone()` on it skipped the reference path. Fixed in `cn1_globals.m` and + `nativeMethods.m`, held by `PrimitiveArrayOfArraysIntegrationTest`. +- **The world sweeps only while a body is Continuous.** Box2D finds times of + impact for every dynamic body against every static one by default; Unity + does it for a `Rigidbody2D` set to Continuous and no other. Box2D's switch is + per world, so `PhysicsWorld.sweepIfAnyAsks` turns it on while any body asks. + Left on, a level of Discrete bodies on a tilemap spent 70% of its step in + `World.solveTOI`. `platformer2d`'s pebble is the Continuous body: the frame + it turns back on (150, and 156 when nothing sweeps) is what holds this. +- **A level with a tilemap sorts dozens of broad-phase pairs a step.** + `Arrays.sort` takes a merge buffer above 31 elements, which was most of + what such a level allocated; `BroadPhase.sortPairs` sorts in place + (`BroadPhasePairSortTest`). Contact delivery likewise refills one + `Collision2D` per side of a pair in touch, as Unity's + `reuseCollisionCallbacks` does -- a script must not keep the one it is given. +- **Two images with one file name ship as `Name.png` and `Name~N.png`.** + Resources are flat on a device; a project with `Idle.png` in ten folders is + ordinary, so the name is made distinct rather than warned about. +- **`unity-compat-project-test.sh` resolves from `MAVEN_REPO_LOCAL`**, and + reads `~/.m2` when it is unset: export it to the checkout's `.m2-repo` after + installing `core`, both Unity modules and both build plugins there. +- **Clean room.** Unity's reference source and Unity's own assemblies are + never read, decompiled or shipped. The `UnityEngine.dll` in the references + jar is compiled from our own `src/main/csharp`, every body a bare `throw`; the + API comes from Unity's public documentation and from observed behaviour. +- **`maven/integration-tests/unity-compat-test.sh`** runs every sample under + `scripts/unity-compat-samples` on the JVM, natively and on JavaScript and + compares traces; **`unity-compat-project-test.sh`** imports one into an + archetype application, builds it with the Maven goal, converts it to + Gradle and holds every target's staged upload to the same entries from both + (`.github/workflows/unity-compat.yml`). The first also runs each scene + sample through the real view path without a window + (`scripts/unity-compat-samples/player/offscreen-check.sh`), so it needs the + JavaSE port jar and compiled `core-unittests` test classes and stops with the + build command when either is missing. +- **`netstandard.dll` in the references jar is Microsoft's**, from the NuGet + package `NETStandard.Library.Ref` (MIT). Its `LICENSE.TXT` and + `THIRD-PARTY-NOTICES.TXT` are kept verbatim in + `maven/unity-compat/third-party` and packed into the jar under + `META-INF/licenses`; the root `NOTICE` has the entry. The .NET SDK does not + bundle that package, so the first build of `maven/unity-compat` on a machine + restores it from nuget.org (later ones read `NUGET_PACKAGES`, offline). An + application build never restores anything: its generated project turns the + implicit framework reference off and names the three assemblies by path. + +#### The screenshot suite's Unity gallery is prebuilt, once, on Linux + +`scripts/hellocodenameone` compiles `scripts/unity-compat-samples/gallery` +(`compile-unity` in `common/pom.xml`) for the `UnityCompat*` screenshot tests, +so every job that builds the application would need the .NET SDK and both +Unity modules. None has them. One Linux job per workflow run does: + +- **`.github/workflows/unity-gallery-prebuilt.yml`** (reusable) builds the + plugin and the two modules, runs the application's common module to + `generate-sources`, and uploads what `scripts/unity-gallery-prebuilt.sh pack` + writes as the artifact `unity-gallery-prebuilt`: the `codenameone-unity-compat` + and `codenameone-cil-translator` artifacts plus `common/target/unity` and + `common/target/generated-sources/unity`. The archive is cached on a hash of + everything that can change a byte of it, so most runs only upload it again. +- **`.github/actions/unity-gallery-prebuilt`** downloads and unpacks it. + `compile-unity` then finds a matching content digest and never starts + `dotnet`. + +Traps: + +- **A new job that builds the application must use both**: a job that + `uses:` the workflow, `needs:` on it, and the action after everything that + fills the local Maven repository and before the first build of the + application. Without them the job fails in `compile-unity` with ".NET SDK, + which was not found", naming the gallery directory. +- **A consumer must not build the Unity jars itself**, or install them from + anywhere else. The digest covers every class of the runtime, the references + and the translator, so output is good only with the archive's own jars. When + they differ the failure adds "does not match these sources or jars" -- that, + not a missing SDK, is the thing to fix. `state.txt` records a digest per + part under the whole one, and the message names the part that differs and, + within it, the first entry: read that before guessing. +- **The translator's class path is its runtime scope and nothing more** + (`CompileUnityMojo.resolve`). Maven's legacy resolver also hands back a + pom's test-scoped dependencies unless it is filtered; that once put the + consumer's own `codenameone-core` jar into the digest, which a Linux job + rebuilds byte for byte and a Windows one does not, so every Windows consumer + refused an archive that was good. +- **A hand-assembled translation takes the application's libraries from + `common/target/translation-libs`**, which the application's own pom stages + from its runtime class path in `process-classes`. The native Linux and + Windows translations (`vm/tests`, `helloSuiteLibraryClasses`) and the + JavaScript build script read it; a list of class directories per workflow + is how the Unity runtime was missing from three of them. +- **The artifact name is fixed, so the workflow is called once per run.** A + workflow that is itself called (`scripts-ios-native.yml`, `ios-packaging.yml`, + `scripts-mac-catalyst.yml`) takes `unity_prepared` and skips its own call + when `scripts-ios.yml` already made one. +- **Anything new that determines the archive goes into the cache key** in the + reusable workflow. A stale cached archive is not refused downstream -- it is + self-consistent -- it is simply used. +- **Every workflow that builds the application lists the gallery, both + modules, the script, the workflow and the action in its `paths`**, and + `.github/ci/apple-checks.json` lists them for the four Apple suites. +- **`unpack` after `mvn clean` on the application, never before**: the output + lives under `common/target`. +- `unity-compat.yml`'s `unity-gallery-consumer` job is the end-to-end test: + it unpacks into a job that built neither module, builds the common module + with an SDK setting that names nothing, and then checks that a stale + `state.txt` is refused by name. + ### Integration Tests Located in `maven/integration-tests/`: diff --git a/CodenameOne/src/com/codename1/charts/ChartComponent.java b/CodenameOne/src/com/codename1/charts/ChartComponent.java index af6e4b98b54..1ab8c82fa0c 100644 --- a/CodenameOne/src/com/codename1/charts/ChartComponent.java +++ b/CodenameOne/src/com/codename1/charts/ChartComponent.java @@ -274,14 +274,28 @@ public void paint(Graphics g) { boolean oldAntialias = g.isAntiAliased(); g.setAntiAliased(true); - boolean transformed = false; - if (getTransform() != null) { - transformed = true; + // The transform the caller painted with is put back exactly as it was + // found, whether or not this component has one of its own. A chart + // turns the canvas for a rotated label and turns it back, and turning + // back is not exact: the angle is a float, so the round trip leaves a + // matrix that is the identity to fifteen digits and not to the + // sixteenth. Whatever is painted after the chart then runs under a + // transform that is no longer the identity, which a port may draw + // differently -- the JavaScript port draws text as DOM nodes only + // under the identity -- so a chart changed how the rest of its form + // was drawn. Asked for only where the platform has transforms: one + // that has none cannot be left with a residue either. + boolean transformed = getTransform() != null; + boolean restore = transformed || g.isTransformSupported(); + if (restore) { + // One object for every paint. It is written here and read back at + // the end of this same call, and nothing in between is handed it. if (tmpTransform == null) { tmpTransform = Transform.makeIdentity(); } g.getTransform(tmpTransform); - + } + if (transformed) { if (currentTransform == null) { currentTransform = Transform.makeIdentity(); } else { @@ -301,13 +315,14 @@ public void paint(Graphics g) { } - util.paintChart(g, chart, getBounds(), getAbsoluteX(), getAbsoluteY()); - - if (transformed) { - g.setTransform(tmpTransform); + try { + util.paintChart(g, chart, getBounds(), getAbsoluteX(), getAbsoluteY()); + } finally { + if (restore) { + g.setTransform(tmpTransform); + } + g.setAntiAliased(oldAntialias); } - - g.setAntiAliased(oldAntialias); } /// Converts screen coordinates to chart coordinates. diff --git a/CodenameOne/src/com/codename1/charts/compat/Canvas.java b/CodenameOne/src/com/codename1/charts/compat/Canvas.java index 520811e9b61..eb3c151357d 100644 --- a/CodenameOne/src/com/codename1/charts/compat/Canvas.java +++ b/CodenameOne/src/com/codename1/charts/compat/Canvas.java @@ -215,6 +215,36 @@ public void rotate(float angle, float x, float y) { g.setTransform(t); } + /// Copies the transform in effect into `out`, so that a temporary rotation, + /// scale or translation can be undone by handing the same object to + /// `#setTransform(Transform)`. + /// + /// Undo a temporary transform this way and never by applying its inverse. + /// The angle of a rotation is a `float`, its sine and cosine are not exact, + /// and rotating by `a` and then by `-a` leaves a matrix that is identity to + /// fifteen digits and not to the sixteenth. Nothing shows on a platform that + /// only multiplies by it, but the JavaScript port draws text as DOM nodes + /// under an identity transform and on the canvas under any other, with + /// different metrics: one rotated axis title moved every label painted after + /// it, the form's own title included, by a few pixels. + /// + /// #### Parameters + /// + /// - `out`: receives the current transform + public void getTransform(Transform out) { + g.getTransform(out); + } + + /// Replaces the transform in effect, typically with one captured by + /// `#getTransform(Transform)`. + /// + /// #### Parameters + /// + /// - `t`: the transform to install + public void setTransform(Transform t) { + g.setTransform(t); + } + public void scale(float x, float y) { Transform t = g.getTransform(); t.translate(bounds.getX(), bounds.getY()); diff --git a/CodenameOne/src/com/codename1/charts/views/XYChart.java b/CodenameOne/src/com/codename1/charts/views/XYChart.java index 78382db84ab..1ee8b24c23e 100644 --- a/CodenameOne/src/com/codename1/charts/views/XYChart.java +++ b/CodenameOne/src/com/codename1/charts/views/XYChart.java @@ -30,6 +30,7 @@ import com.codename1.charts.util.MathHelper; import com.codename1.ui.Component; import com.codename1.ui.Font; +import com.codename1.ui.Transform; import com.codename1.ui.geom.Rectangle; import com.codename1.ui.geom.Rectangle2D; @@ -56,6 +57,15 @@ public abstract class XYChart extends AbstractChart { private float mTranslate; /// The canvas center point. private Point mCenter; + /// The transform that was in effect before `#drawText` rotated the canvas, + /// reused between calls. See `Canvas#getTransform(Transform)` for why a + /// rotation is undone by restoring this and not by rotating back. + private Transform mTextTransform; + /// The transform that was in effect before `#transform` turned the canvas + /// for the vertical orientation, or null while the canvas is not turned. + private Transform mOrientationTransform; + /// Storage for `#mOrientationTransform`, reused between paints. + private Transform mOrientationTransformStore; /// The visible chart area, in screen coordinates. private Rectangle mScreenR; /// The clickable areas for all points. The array index is the series index, @@ -694,12 +704,19 @@ protected void drawText(Canvas canvas, String text, float x, float y, Paint pain float extraAngle) { float angle = -mRenderer.getOrientation().getAngle() + extraAngle; if (angle != 0) { + if (mTextTransform == null) { + mTextTransform = Transform.makeIdentity(); + } + canvas.getTransform(mTextTransform); // canvas.scale(1 / mScale, mScale); canvas.rotate(angle, x, y); } drawString(canvas, text, x, y, paint); if (angle != 0) { - canvas.rotate(-angle, x, y); + // Restored, not rotated back by -angle: the round trip is not exact + // in float arithmetic and would leave a transform that is no longer + // the identity for everything painted after this text. + canvas.setTransform(mTextTransform); // canvas.scale(mScale, 1 / mScale); } } @@ -719,10 +736,22 @@ private void transform(Canvas canvas, float angle, boolean inverse) { return; } if (inverse) { + if (mOrientationTransform != null) { + // The exact transform the forward call replaced; the computed + // inverse below only approximates it. + canvas.setTransform(mOrientationTransform); + mOrientationTransform = null; + return; + } canvas.scale(1 / mScale, mScale); canvas.translate(mTranslate, -mTranslate); canvas.rotate(-angle, mCenter.getX(), mCenter.getY()); } else { + if (mOrientationTransformStore == null) { + mOrientationTransformStore = Transform.makeIdentity(); + } + canvas.getTransform(mOrientationTransformStore); + mOrientationTransform = mOrientationTransformStore; canvas.rotate(angle, mCenter.getX(), mCenter.getY()); canvas.translate(-mTranslate, mTranslate); canvas.scale(mScale, 1 / mScale); diff --git a/CodenameOne/src/com/codename1/gaming/GameView.java b/CodenameOne/src/com/codename1/gaming/GameView.java index 000dfb9d42b..1e313b70ce0 100644 --- a/CodenameOne/src/com/codename1/gaming/GameView.java +++ b/CodenameOne/src/com/codename1/gaming/GameView.java @@ -25,6 +25,7 @@ import com.codename1.gpu.GraphicsDevice; import com.codename1.gpu.Light; import com.codename1.gpu.RenderView; +import com.codename1.ui.Image; import com.codename1.ui.events.ActionEvent; import com.codename1.ui.events.ActionListener; import com.codename1.ui.geom.Rectangle; @@ -82,6 +83,15 @@ public abstract class GameView extends RenderView implements SpriteRenderer.Upda private ActionListener dragListener; private ActionListener releaseListener; private boolean formListenersAdded; + /// Set once the form has delivered a pointer event to this view directly, + /// with every finger in it. See `#addFormPointerListeners()`. + private boolean touchesArrive; + /// Whether a press reached the game and its release has not: by either + /// path, the form's listeners or this view's own callbacks. The form's + /// listeners hear every pointer event on the form, the host's own + /// buttons beside the view included, and only a gesture the game began + /// may be moved or ended through them. + private boolean gesture; public GameView() { super(new SpriteRenderer()); @@ -138,6 +148,26 @@ public void removeModel(Model model) { ((SpriteRenderer) getRenderer()).removeModel(model); } + /// Releases the texture the view keeps for an image that is no longer + /// drawn. The view uploads each image a sprite shows once and keeps the + /// texture for as long as it lives, so an image that is made at run time + /// and then replaced -- a text painted into an image, for one -- has to be + /// released, or its texture stays. Safe to call from `update`; an image + /// that is drawn again later is uploaded again, and one the view has no + /// texture for is ignored. See `SpriteRenderer#releaseTexture(Image)`. + /// + /// #### Parameters + /// + /// - `image`: the image whose texture is no longer needed + public void releaseTexture(Image image) { + ((SpriteRenderer) getRenderer()).releaseTexture(image); + } + + /// The number of images the view holds a texture for at the moment. + public int getTextureCount() { + return ((SpriteRenderer) getRenderer()).getTextureCount(); + } + /// Override to allocate GPU resources (meshes, textures, `Model`s) once the GPU /// device is ready. Invoked once on the render thread before the first frame -- /// the only place you can call `com.codename1.gpu.Primitives` / @@ -225,6 +255,14 @@ public double getInterpolationAlpha() { @Override protected void initComponent() { super.initComponent(); + // What the view hosts -- the GPU surface, or the placeholder that stands + // in for it -- is the view as far as a touch goes. A form hands a pointer + // event to the deepest component under it, which would be that child, and + // it does nothing with one; marked this way the form passes it on to its + // parent, and the multi-touch callbacks of this class see every finger. + for (int i = 0; i < getComponentCount(); i++) { + getComponentAt(i).setIgnorePointerEvents(true); + } addFormPointerListeners(); if (running) { setContinuous(true); @@ -236,15 +274,27 @@ protected void initComponent() { @Override protected void deinitialize() { removeFormPointerListeners(); + // A gesture cut off by the view leaving its form is not resumed by the + // first release on the form it is shown on next. + gesture = false; super.deinitialize(); } - /// The GPU surface is hosted in a native peer that swallows the platform's - /// pointer events before they reach this component, so the usual - /// `#pointerPressed(int[], int[])` callbacks never fire over the surface. Instead - /// we listen at the form level (those listeners fire for every pointer event, - /// regardless of which component is hit) and route the touches to the on-screen - /// controls ourselves. + /// Form-level listeners fire for every pointer event, whichever component is + /// hit, but they carry the first finger only. They are what feeds the view + /// until `#pointerPressed(int[], int[])` and its siblings have been seen to + /// arrive -- those carry every finger, and once they do, a press or a drag + /// heard here as well would take the on-screen controls back to one finger + /// between two reports of all of them. A release is routed whenever a gesture + /// of the game's is in progress: it means the last finger lifted, saying so + /// twice changes nothing, and it must not be lost when the form delivers it to + /// another component. + /// + /// They hear the form's other components too. A press is the game's only + /// inside the view's bounds; a press on a host's button beside the view used + /// to be handed to the game as well, as a touch outside its area. A drag or + /// a release goes to the game only when the press before it did, and then + /// wherever it is, so that a finger that slides off the view still lets go. private void addFormPointerListeners() { if (formListenersAdded) { return; @@ -257,19 +307,25 @@ private void addFormPointerListeners() { pressListener = new ActionListener() { @Override public void actionPerformed(ActionEvent e) { - routeFormTouch(e.getX(), e.getY(), true, true, false); + if (!touchesArrive && contains(e.getX(), e.getY())) { + routeFormTouch(e.getX(), e.getY(), true, true, false); + } } }; dragListener = new ActionListener() { @Override public void actionPerformed(ActionEvent e) { - routeFormTouch(e.getX(), e.getY(), true, false, false); + if (!touchesArrive && gesture) { + routeFormTouch(e.getX(), e.getY(), true, false, false); + } } }; releaseListener = new ActionListener() { @Override public void actionPerformed(ActionEvent e) { - routeFormTouch(e.getX(), e.getY(), false, false, true); + if (gesture) { + routeFormTouch(e.getX(), e.getY(), false, false, true); + } } }; } @@ -297,6 +353,7 @@ private void removeFormPointerListeners() { private void routeFormTouch(int x, int y, boolean down, boolean pressed, boolean released) { int lx = x - getAbsoluteX(); int ly = y - getAbsoluteY(); + gesture = down; controls.onTouches(new int[]{lx}, new int[]{ly}, down); input.pointer(lx, ly, down, pressed, released); } @@ -379,18 +436,51 @@ public void keyReleased(int keyCode) { input.keyUp(keyCode); } + /// A key that is held is held, and nothing more. The display reports a key + /// that stays down as repeating -- 800 ms after the press and then every few + /// milliseconds, by the wall clock -- and a component turns each repeat into a + /// press and a release unless it says otherwise. For a text field that is a + /// character typed again; here it would let go of a key the player still holds: + /// `GameInput#isKeyDown(int)` went false one repeat interval into every held + /// direction on a port that does not re-send the press itself, and a subclass + /// that forwards `#keyReleased(int)` saw the key come up. + @Override + public void keyRepeated(int keyCode) { + // Deliberately empty: see above. + } + + /// A form hands a press to the component under it through this overload and + /// never through the one that takes arrays, which only a drag arrives by. + /// Unless it is passed on, the first drag marks the view as hearing its + /// touches directly, the form's own listeners stop feeding it presses, and + /// from then on no press reaches the game at all: a click after the pointer + /// had once been dragged did nothing. + @Override + public void pointerPressed(int x, int y) { + pointerPressed(new int[] {x}, new int[] {y}); + } + + /// The release of `#pointerPressed(int, int)`, for the same reason. + @Override + public void pointerReleased(int x, int y) { + pointerReleased(new int[] {x}, new int[] {y}); + } + @Override public void pointerPressed(int[] x, int[] y) { + touchesArrive = true; routeTouches(x, y, true, true, false); } @Override public void pointerDragged(int[] x, int[] y) { + touchesArrive = true; routeTouches(x, y, true, false, false); } @Override public void pointerReleased(int[] x, int[] y) { + touchesArrive = true; routeTouches(x, y, false, false, true); } @@ -402,6 +492,7 @@ private void routeTouches(int[] x, int[] y, boolean down, boolean pressed, boole int oy = getAbsoluteY(); int[] lx = new int[x.length]; int[] ly = new int[y.length]; + gesture = down; for (int i = 0; i < x.length; i++) { lx[i] = x[i] - ox; ly[i] = y[i] - oy; diff --git a/CodenameOne/src/com/codename1/gaming/SpriteRenderer.java b/CodenameOne/src/com/codename1/gaming/SpriteRenderer.java index 9aa1462ebce..405d0bd6ac0 100644 --- a/CodenameOne/src/com/codename1/gaming/SpriteRenderer.java +++ b/CodenameOne/src/com/codename1/gaming/SpriteRenderer.java @@ -84,6 +84,8 @@ interface Updatable { private RenderState spriteState2D; private RenderState spriteState3D; private Map textures; + // images whose textures were asked back; see releaseTexture + private final List released = new ArrayList(); private Image discImage; private Image ringImage; private Map labelImages; @@ -140,6 +142,55 @@ public int getModelCount() { return models.size(); } + /// Releases the texture this renderer keeps for an image, and forgets the + /// image. + /// + /// The renderer uploads every image a sprite is drawn with once, and keeps + /// the texture until it is disposed itself. That is right for the art of a + /// game, and wrong for an image that is made, shown for a while and never + /// shown again -- a text painted into an image, a generated frame: each one + /// would hold its texture, and the image with it, for as long as the view + /// lives. Call this when such an image is no longer drawn. + /// + /// The GPU is not touched here, since only a renderer callback has the + /// device: the texture is disposed at the start of the next frame, before + /// anything is drawn. So it may be called from the frame itself (`GameView`'s + /// `update`), and an image released and drawn again -- in the same frame or + /// any later one -- is simply uploaded again. An image the renderer has no + /// texture for, or null, is ignored. + /// + /// Like every change to the scene, it belongs on the thread that runs the + /// frame. + /// + /// #### Parameters + /// + /// - `image`: the image whose texture is no longer needed + public void releaseTexture(Image image) { + if (image != null && textures != null && textures.containsKey(image)) { + released.add(image); + } + } + + /// The number of images this renderer holds a texture for at the moment: + /// those drawn since the device was created and not released since. + public int getTextureCount() { + return textures == null ? 0 : textures.size(); + } + + /// Disposes the textures that were released since the last frame. An image + /// released twice is found in the map only the first time, so no texture is + /// handed to the device twice. + private void disposeReleased(GraphicsDevice device) { + int n = released.size(); + for (int i = 0; i < n; i++) { + Texture t = (Texture) textures.remove(released.get(i)); + if (t != null) { + device.dispose(t); + } + } + released.clear(); + } + /// The ARGB color the framebuffer is cleared to each frame. public void setClearColor(int argb) { this.clearColor = argb; @@ -189,6 +240,8 @@ public void onInit(GraphicsDevice device) { .setCullMode(RenderState.CullMode.NONE); material = new Material(Material.Type.SPRITE).setRenderState(spriteState2D); textures = new HashMap(); + // a new device: what was released belonged to the one before it + released.clear(); labelImages = new HashMap(); discImage = circleTexture(128, 0f, 1f); ringImage = circleTexture(128, 0.72f, 1f); @@ -226,6 +279,9 @@ public void onFrame(GraphicsDevice device) { updatable.frame(dt); } scene.update(dt); + if (!released.isEmpty()) { + disposeReleased(device); + } device.clear(clearColor, true, true); // configure the GPU camera from the GameCamera each frame so a moving 3D @@ -388,6 +444,8 @@ public void onDispose(GraphicsDevice device) { } textures.clear(); } + // their textures went with the rest, just above + released.clear(); if (quad != null) { device.dispose(quad.getVertices()); device.dispose(quad.getIndices()); diff --git a/CodenameOne/src/com/codename1/gaming/physics/box2d/collision/broadphase/BroadPhase.java b/CodenameOne/src/com/codename1/gaming/physics/box2d/collision/broadphase/BroadPhase.java index a9375a590b6..dada8b5b3ec 100644 --- a/CodenameOne/src/com/codename1/gaming/physics/box2d/collision/broadphase/BroadPhase.java +++ b/CodenameOne/src/com/codename1/gaming/physics/box2d/collision/broadphase/BroadPhase.java @@ -23,8 +23,6 @@ ******************************************************************************/ package com.codename1.gaming.physics.box2d.collision.broadphase; -import java.util.Arrays; - import com.codename1.gaming.physics.box2d.callbacks.DebugDraw; import com.codename1.gaming.physics.box2d.callbacks.PairCallback; import com.codename1.gaming.physics.box2d.callbacks.TreeCallback; @@ -169,7 +167,7 @@ public final void updatePairs(PairCallback callback) { m_moveCount = 0; // Sort the pair buffer to expose duplicates. - Arrays.sort(m_pairBuffer, 0, m_pairCount); + sortPairs(m_pairBuffer, m_pairCount); // Send the pairs back to the client. int i = 0; @@ -197,6 +195,28 @@ public final void updatePairs(PairCallback callback) { // m_tree.rebalance(Settings.TREE_REBALANCE_STEPS); } + /// Sorts the first `count` pairs by their proxy ids, in place. `Arrays.sort` takes a merge + /// buffer for anything over a few dozen elements, and this runs on every step of a world + /// with many moving proxies. A shell sort needs none, and which of two equal pairs comes + /// first is of no consequence: the caller skips duplicates. + static void sortPairs(Pair[] pairs, int count) { + int gap = 1; + while (gap < count / 3) { + gap = gap * 3 + 1; + } + for (; gap >= 1; gap /= 3) { + for (int i = gap; i < count; i++) { + Pair moved = pairs[i]; + int j = i; + while (j >= gap && pairs[j - gap].compareTo(moved) > 0) { + pairs[j] = pairs[j - gap]; + j -= gap; + } + pairs[j] = moved; + } + } + } + /// Query an AABB for overlapping proxies. The callback class is called for each proxy that /// overlaps the supplied AABB. /// diff --git a/CodenameOne/src/com/codename1/gaming/physics/box2d/dynamics/World.java b/CodenameOne/src/com/codename1/gaming/physics/box2d/dynamics/World.java index 2e2daa1cba9..20fb6521db0 100644 --- a/CodenameOne/src/com/codename1/gaming/physics/box2d/dynamics/World.java +++ b/CodenameOne/src/com/codename1/gaming/physics/box2d/dynamics/World.java @@ -211,6 +211,14 @@ public Contact popContact(Fixture fixtureA, int indexA, Fixture fixtureB, int in final ShapeType type2 = fixtureB.getType(); final ContactRegister reg = contactStacks[type1.ordinal()][type2.ordinal()]; + // Codename One: two shapes with no collision routine between them -- a chain + // against a chain, an edge against an edge or a chain -- have no register at + // all, and Box2D's answer for such a pair is no contact. Reading the creator + // of a null register threw here the moment a dynamic body made of chains + // came near another chain. + if (reg == null) { + return null; + } final IDynamicStack creator = reg.creator; if (creator != null) { if (reg.primary) { diff --git a/CodenameOne/src/com/codename1/gpu/RenderState.java b/CodenameOne/src/com/codename1/gpu/RenderState.java index 93182ded768..5ca2e7bfc76 100644 --- a/CodenameOne/src/com/codename1/gpu/RenderState.java +++ b/CodenameOne/src/com/codename1/gpu/RenderState.java @@ -6,6 +6,19 @@ * published by the Free Software Foundation. Codename One designates this * particular file as subject to the "Classpath" exception as provided * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.gpu; @@ -14,12 +27,27 @@ /// geometry (depth test and write on, no blending, back faces culled). public final class RenderState { /// The alpha blending mode applied when a fragment is written. + /// + /// A fragment's color is a material's color multiplied by its texture, in + /// straight (not premultiplied) alpha: a material colored `0x66000000` over + /// a white texture is black at 40%, and drawn with `ALPHA` over an opaque + /// destination `d` it leaves `0.6 * d`. + /// + /// The framebuffer's own alpha channel is not blended the way its color + /// is. In both blending modes it accumulates coverage, + /// `alpha = source + destination * (1 - source)`, so a frame cleared to an + /// opaque color stays opaque whatever is blended onto it. A backend that + /// weights the alpha by the same factor as the color leaves `0.76` under a + /// fragment at 40%, and where the frame is composited or read back with + /// its alpha, what is behind the view shows through the fragment. public enum BlendMode { /// No blending; the fragment overwrites the destination. NONE, - /// Standard source-over alpha blending. + /// Standard source-over alpha blending: + /// `color = source * sourceAlpha + destination * (1 - sourceAlpha)`. ALPHA, - /// Additive blending, useful for particles and glows. + /// Additive blending, useful for particles and glows: + /// `color = source * sourceAlpha + destination`. ADDITIVE } diff --git a/CodenameOne/src/com/codename1/impl/CodenameOneImplementation.java b/CodenameOne/src/com/codename1/impl/CodenameOneImplementation.java index 545ad9b28e5..5f153261e54 100644 --- a/CodenameOne/src/com/codename1/impl/CodenameOneImplementation.java +++ b/CodenameOne/src/com/codename1/impl/CodenameOneImplementation.java @@ -3768,7 +3768,17 @@ protected void pointerReleased(final int[] x, final int[] y) { if (dragActivationCounter == 0 && x[0] != pointerPressedX && y[0] != pointerPressedY) { hasDragStarted(pointerPressedX, pointerPressedY); if (hasDragStarted(x, y)) { + // The single point overloads all hand over the same two arrays, so a + // release that came through pointerReleased(int, int) IS those arrays, + // and the drag synthesized at the press position below writes over it: + // the drag meant for the release point and then the release itself + // were both delivered at the press point, and the component under the + // real release point never heard it. + int releasedX = x[0]; + int releasedY = y[0]; pointerDragged(pointerPressedX, pointerPressedY); + x[0] = releasedX; + y[0] = releasedY; pointerDragged(x, y); } } diff --git a/NOTICE b/NOTICE index dc50ab27df2..63816df6176 100644 --- a/NOTICE +++ b/NOTICE @@ -129,3 +129,46 @@ ASM is distributed under the BSD 3-Clause license: CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================================ +.NET Standard reference assembly +================================================================================ + +The "references" artifact of maven/unity-compat (codenameone-unity-compat, +classifier references) holds the assemblies the C# scripts of a Unity project +are compiled against. One of them, netstandard.dll, is the .NET Standard 2.1 +reference assembly from the NuGet package NETStandard.Library.Ref 2.1.0 +(https://www.nuget.org/packages/NETStandard.Library.Ref/2.1.0), redistributed +unmodified. It declares the API of the .NET base library and contains no +implementation; it is read by the C# compiler and the translator at build time +and is never part of an application. + +The package's LICENSE.TXT and THIRD-PARTY-NOTICES.TXT are kept verbatim in +maven/unity-compat/third-party/NETStandard.Library.Ref and ship inside the +artifact under META-INF/licenses/NETStandard.Library.Ref. + +NETStandard.Library.Ref is distributed under the MIT license: + + The MIT License (MIT) + + Copyright (c) .NET Foundation and Contributors + + All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. diff --git a/Ports/Android/src/com/codename1/impl/android/AndroidGraphicsDevice.java b/Ports/Android/src/com/codename1/impl/android/AndroidGraphicsDevice.java index 6b3552cc3c8..4d350d32aa1 100644 --- a/Ports/Android/src/com/codename1/impl/android/AndroidGraphicsDevice.java +++ b/Ports/Android/src/com/codename1/impl/android/AndroidGraphicsDevice.java @@ -6,6 +6,19 @@ * published by the Free Software Foundation. Codename One designates this * particular file as subject to the "Classpath" exception as provided * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.impl.android; @@ -330,10 +343,17 @@ private void applyRenderState(RenderState rs) { GLES20.glDisable(GLES20.GL_BLEND); } else { GLES20.glEnable(GLES20.GL_BLEND); + // The alpha channel accumulates coverage rather than being + // weighted by itself, so an opaque frame stays opaque + // (RenderState.BlendMode's contract). The default surface has no + // alpha channel and showed nothing of this; one that does -- a + // translucent surface, a frame read back -- would have. if (blend == RenderState.BlendMode.ADDITIVE) { - GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE); + GLES20.glBlendFuncSeparate(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE, + GLES20.GL_ONE, GLES20.GL_ONE_MINUS_SRC_ALPHA); } else { - GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA); + GLES20.glBlendFuncSeparate(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA, + GLES20.GL_ONE, GLES20.GL_ONE_MINUS_SRC_ALPHA); } } diff --git a/Ports/JavaSE/src/com/codename1/impl/javase/JavaSEGLDevice.java b/Ports/JavaSE/src/com/codename1/impl/javase/JavaSEGLDevice.java index 67ed1155b79..66e5715bc62 100644 --- a/Ports/JavaSE/src/com/codename1/impl/javase/JavaSEGLDevice.java +++ b/Ports/JavaSE/src/com/codename1/impl/javase/JavaSEGLDevice.java @@ -6,6 +6,19 @@ * published by the Free Software Foundation. Codename One designates this * particular file as subject to the "Classpath" exception as provided * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.impl.javase; @@ -311,10 +324,17 @@ private void applyRenderState(RenderState rs) { gl.glDisable(GL2ES2.GL_BLEND); } else { gl.glEnable(GL2ES2.GL_BLEND); + // The alpha channel accumulates coverage rather than being + // weighted by itself, so an opaque frame stays opaque + // (RenderState.BlendMode's contract). The frame is read back + // with its alpha and painted over the form: with one glBlendFunc + // a 40% quad left 0.76 there and the form showed through it. if (blend == RenderState.BlendMode.ADDITIVE) { - gl.glBlendFunc(GL2ES2.GL_SRC_ALPHA, GL2ES2.GL_ONE); + gl.glBlendFuncSeparate(GL2ES2.GL_SRC_ALPHA, GL2ES2.GL_ONE, + GL2ES2.GL_ONE, GL2ES2.GL_ONE_MINUS_SRC_ALPHA); } else { - gl.glBlendFunc(GL2ES2.GL_SRC_ALPHA, GL2ES2.GL_ONE_MINUS_SRC_ALPHA); + gl.glBlendFuncSeparate(GL2ES2.GL_SRC_ALPHA, GL2ES2.GL_ONE_MINUS_SRC_ALPHA, + GL2ES2.GL_ONE, GL2ES2.GL_ONE_MINUS_SRC_ALPHA); } } diff --git a/Ports/JavaSE/src/com/codename1/impl/javase/JavaSESoftwareDevice.java b/Ports/JavaSE/src/com/codename1/impl/javase/JavaSESoftwareDevice.java index f266104e5f3..e88c59ad745 100644 --- a/Ports/JavaSE/src/com/codename1/impl/javase/JavaSESoftwareDevice.java +++ b/Ports/JavaSE/src/com/codename1/impl/javase/JavaSESoftwareDevice.java @@ -6,6 +6,19 @@ * published by the Free Software Foundation. Codename One designates this * particular file as subject to the "Classpath" exception as provided * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. */ package com.codename1.impl.javase; @@ -527,13 +540,17 @@ private static int blendPixel(int dst, int src, boolean additive, float srcAlpha int dr = (dst >> 16) & 0xff; int dg = (dst >> 8) & 0xff; int db = dst & 0xff; + float sa = srcAlpha; if (additive) { + // The light added is weighted by the source alpha, as on every + // GPU backend (source factor SRC_ALPHA, destination ONE): a + // particle fading out adds less and less. Unweighted, it stayed + // at full strength here until the frame it vanished. return (0xff << 24) - | (clamp255(dr + sr) << 16) - | (clamp255(dg + sg) << 8) - | clamp255(db + sb); + | (clamp255((int) (dr + sr * sa + 0.5f)) << 16) + | (clamp255((int) (dg + sg * sa + 0.5f)) << 8) + | clamp255((int) (db + sb * sa + 0.5f)); } - float sa = srcAlpha; float ia = 1.0f - sa; int rr = clamp255((int) (sr * sa + dr * ia + 0.5f)); int rg = clamp255((int) (sg * sa + dg * ia + 0.5f)); diff --git a/Ports/JavaScriptPort/src/main/java/com/codename1/html5/js/webgl/WebGLRenderingContext.java b/Ports/JavaScriptPort/src/main/java/com/codename1/html5/js/webgl/WebGLRenderingContext.java index 76f4c44fad9..f17ed58dec2 100644 --- a/Ports/JavaScriptPort/src/main/java/com/codename1/html5/js/webgl/WebGLRenderingContext.java +++ b/Ports/JavaScriptPort/src/main/java/com/codename1/html5/js/webgl/WebGLRenderingContext.java @@ -53,6 +53,7 @@ public interface WebGLRenderingContext extends JSObject { void disable(int cap); void depthMask(boolean flag); void blendFunc(int sfactor, int dfactor); + void blendFuncSeparate(int srcRGB, int dstRGB, int srcAlpha, int dstAlpha); void cullFace(int mode); WebGLShader createShader(int type); diff --git a/Ports/JavaScriptPort/src/main/java/com/codename1/impl/html5/BufferedGraphics.java b/Ports/JavaScriptPort/src/main/java/com/codename1/impl/html5/BufferedGraphics.java index a8e05e3b98f..e959e400574 100644 --- a/Ports/JavaScriptPort/src/main/java/com/codename1/impl/html5/BufferedGraphics.java +++ b/Ports/JavaScriptPort/src/main/java/com/codename1/impl/html5/BufferedGraphics.java @@ -1246,8 +1246,23 @@ void setPromotionSuspended(boolean value) { // Shared by text runs and persistent editors: DOM rectangles cannot represent // a transformed paint context or a nonrectangular clip. + // + // "Transformed" is judged within a tolerance and not exactly. Rotating the + // graphics and rotating it back leaves a matrix that is the identity to + // fifteen digits, and an exact test called that a transform: every string + // painted after one rotated label left the DOM layer for the canvas, where + // it is measured differently. JavaScriptTransformTolerance has the bounds + // and why nothing drawn can tell the difference. boolean supportsNativeTextOverlay() { - return !clipEmpty && !isClipShape && (transform == null || transform.isIdentity()); + return !clipEmpty && !isClipShape && isIdentityForText(transform); + } + + private static boolean isIdentityForText(Transform t) { + if (t == null || t.isIdentity()) { + return true; + } + Object nativeTransform = t.getNativeTransform(); + return nativeTransform instanceof JSAffineTransform && ((JSAffineTransform) nativeTransform).isNearIdentity(); } /** diff --git a/Ports/JavaScriptPort/src/main/java/com/codename1/impl/html5/HTML5GraphicsDevice.java b/Ports/JavaScriptPort/src/main/java/com/codename1/impl/html5/HTML5GraphicsDevice.java index 20fed32b166..f2c3c9d1d4e 100644 --- a/Ports/JavaScriptPort/src/main/java/com/codename1/impl/html5/HTML5GraphicsDevice.java +++ b/Ports/JavaScriptPort/src/main/java/com/codename1/impl/html5/HTML5GraphicsDevice.java @@ -73,9 +73,6 @@ class HTML5GraphicsDevice extends GraphicsDevice { private static final int LINE_STRIP = 0x0003; private static final int TRIANGLES = 0x0004; private static final int TRIANGLE_STRIP = 0x0005; - private static final int SRC_ALPHA = 0x0302; - private static final int ONE_MINUS_SRC_ALPHA = 0x0303; - private static final int ONE = 1; private static final int CULL_FACE = 0x0B44; private static final int DEPTH_TEST = 0x0B71; private static final int BLEND = 0x0BE2; @@ -382,11 +379,13 @@ private void applyRenderState(RenderState state) { gl.disable(BLEND); } else { gl.enable(BLEND); - if (blend == RenderState.BlendMode.ADDITIVE) { - gl.blendFunc(SRC_ALPHA, ONE); - } else { - gl.blendFunc(SRC_ALPHA, ONE_MINUS_SRC_ALPHA); - } + // Separate factors for the alpha channel, so that an opaque + // frame stays opaque: see JavaScriptBlendFactors. + boolean additive = blend == RenderState.BlendMode.ADDITIVE; + gl.blendFuncSeparate(JavaScriptBlendFactors.sourceColor(additive), + JavaScriptBlendFactors.destinationColor(additive), + JavaScriptBlendFactors.sourceAlpha(additive), + JavaScriptBlendFactors.destinationAlpha(additive)); } RenderState.CullMode cull = state.getCullMode(); diff --git a/Ports/JavaScriptPort/src/main/java/com/codename1/impl/html5/JavaScriptBlendFactors.java b/Ports/JavaScriptPort/src/main/java/com/codename1/impl/html5/JavaScriptBlendFactors.java new file mode 100644 index 00000000000..3b8aba0919e --- /dev/null +++ b/Ports/JavaScriptPort/src/main/java/com/codename1/impl/html5/JavaScriptBlendFactors.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.html5; + +/// The four WebGL blend factors of a `com.codename1.gpu.RenderState.BlendMode` +/// that blends, as `blendFuncSeparate` takes them. +/// +/// The colour channels and the alpha channel are given different factors on +/// purpose. The colour is weighted by the source alpha; the alpha channel +/// accumulates coverage, `a = sa + da * (1 - sa)`, so a frame cleared opaque +/// stays opaque whatever is blended onto it. +/// +/// One `blendFunc` for all four channels weights the alpha by itself +/// (`sa * sa + da * (1 - sa)`), and a quad at 40% over an opaque clear then +/// leaves 0.76 in the drawing buffer. The WebGL canvas is premultiplied and +/// is drawn onto the display canvas source-over, so a quarter of whatever was +/// there before -- the frame before this one -- stayed in every such pixel, +/// and a black tint meant to give `0.6 * background` settled at +/// `0.6 * background / 0.76` instead. +/// +/// This class has no natives and no dependencies, so that it compiles and is +/// tested alone under plain JUnit (`GpuBlendContractTest`), which runs +/// the blend equation with these factors. +public final class JavaScriptBlendFactors { + /// `gl.ZERO`. + public static final int ZERO = 0; + /// `gl.ONE`. + public static final int ONE = 1; + /// `gl.SRC_ALPHA`. + public static final int SRC_ALPHA = 0x0302; + /// `gl.ONE_MINUS_SRC_ALPHA`. + public static final int ONE_MINUS_SRC_ALPHA = 0x0303; + + private JavaScriptBlendFactors() { + } + + /// The factor of the source colour: its own alpha, in both modes. + /// + /// #### Parameters + /// + /// - `additive`: true for `BlendMode.ADDITIVE`, false for `BlendMode.ALPHA` + public static int sourceColor(boolean additive) { + return SRC_ALPHA; + } + + /// The factor of the destination colour: all of it when adding light, + /// what the source leaves uncovered otherwise. + /// + /// #### Parameters + /// + /// - `additive`: true for `BlendMode.ADDITIVE`, false for `BlendMode.ALPHA` + public static int destinationColor(boolean additive) { + return additive ? ONE : ONE_MINUS_SRC_ALPHA; + } + + /// The factor of the source alpha: one, never the alpha itself. + /// + /// #### Parameters + /// + /// - `additive`: true for `BlendMode.ADDITIVE`, false for `BlendMode.ALPHA` + public static int sourceAlpha(boolean additive) { + return ONE; + } + + /// The factor of the destination alpha: what the source leaves uncovered. + /// + /// #### Parameters + /// + /// - `additive`: true for `BlendMode.ADDITIVE`, false for `BlendMode.ALPHA` + public static int destinationAlpha(boolean additive) { + return ONE_MINUS_SRC_ALPHA; + } +} diff --git a/Ports/JavaScriptPort/src/main/java/com/codename1/impl/html5/JavaScriptTransformTolerance.java b/Ports/JavaScriptPort/src/main/java/com/codename1/impl/html5/JavaScriptTransformTolerance.java new file mode 100644 index 00000000000..15299ab0a30 --- /dev/null +++ b/Ports/JavaScriptPort/src/main/java/com/codename1/impl/html5/JavaScriptTransformTolerance.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.html5; + +/// Decides whether a 2D affine transform is the identity for the purpose of +/// drawing text as DOM nodes above the canvas, which the port does only under +/// an identity transform: a DOM rectangle cannot represent any other. +/// +/// The test cannot be exact. Application and framework code routinely turns +/// the graphics for one thing and turns it back -- `rotate(a)`, draw, +/// `rotate(-a)` -- and that round trip is not exact arithmetic: the angle is +/// a `float`, its sine and cosine are rounded, and what is left differs from +/// the identity in the fifteenth digit. Compared with `===` that residue was +/// "a transform", so every string painted after it moved from the DOM layer +/// to the canvas, whose font metrics differ by a few pixels. One rotated +/// label in a chart moved the title of the form it was on, and nothing in +/// the application said why. +/// +/// So a transform this close to the identity is taken for it. The bounds are +/// set by what could be seen, not by what the residue happens to be: +/// +/// - The four linear entries may be off by `LINEAR_EPSILON`. A point is moved +/// by that times its coordinate, so at 100,000 pixels from the origin the +/// error is a ten-thousandth of a pixel. The residue of a round trip is five +/// orders of magnitude smaller still. +/// - The two translation entries ARE pixels, and may be off by +/// `TRANSLATION_EPSILON` of one. A deliberate sub-pixel translation is a +/// real transform the canvas honours and the DOM layer would not, so this +/// stays far below anything a caller could mean, and far above the residue +/// of rotating about a pivot (the linear residue times the pivot). +/// +/// A value that is not a number is never the identity. +/// +/// This class has no natives and no dependencies, so that it compiles and is +/// tested alone under plain JUnit (`JavaScriptTransformToleranceTest`). +public final class JavaScriptTransformTolerance { + /// How far a scale or shear entry may be from the identity's. + public static final double LINEAR_EPSILON = 1e-9; + /// How far a translation entry may be from zero, in pixels. + public static final double TRANSLATION_EPSILON = 1e-6; + + private JavaScriptTransformTolerance() { + } + + /// Whether the matrix `[m00 m01 m02; m10 m11 m12]` draws everything where + /// the identity would, to within what a display can show. + public static boolean isNearIdentity(double m00, double m10, double m01, double m11, double m02, double m12) { + return within(m00 - 1.0, LINEAR_EPSILON) && within(m11 - 1.0, LINEAR_EPSILON) + && within(m10, LINEAR_EPSILON) && within(m01, LINEAR_EPSILON) + && within(m02, TRANSLATION_EPSILON) && within(m12, TRANSLATION_EPSILON); + } + + // Written so that NaN fails: every comparison with NaN is false. + private static boolean within(double difference, double epsilon) { + return difference <= epsilon && difference >= -epsilon; + } +} diff --git a/Ports/JavaScriptPort/src/main/java/com/codename1/teavm/geom/JSAffineTransform.java b/Ports/JavaScriptPort/src/main/java/com/codename1/teavm/geom/JSAffineTransform.java index e929acdaa13..e378da1c233 100644 --- a/Ports/JavaScriptPort/src/main/java/com/codename1/teavm/geom/JSAffineTransform.java +++ b/Ports/JavaScriptPort/src/main/java/com/codename1/teavm/geom/JSAffineTransform.java @@ -337,6 +337,23 @@ public boolean isIdentity() { } return jsoIsIdentity(inner); } + /// Whether this transform is the identity as far as anything drawn can + /// tell: exactly, or within the residue that turning the graphics and + /// turning it back leaves behind. See + /// `com.codename1.impl.html5.JavaScriptTransformTolerance`, which says why + /// and how close. `#isIdentity()` stays exact, and is what every caller + /// that goes on to multiply by the matrix should ask; this one is for the + /// choice between text in the DOM and text on the canvas, where an exact + /// test let a rounding error decide. A transform with a 4x4 matrix is + /// held to the exact test: nothing two-dimensional leaves one behind. + public boolean isNearIdentity() { + if (m4 != null) { + return JSMatrix4.isIdentity(m4); + } + return com.codename1.impl.html5.JavaScriptTransformTolerance.isNearIdentity(jsoScaleX(inner), + jsoShearY(inner), jsoShearX(inner), jsoScaleY(inner), jsoTranslateX(inner), jsoTranslateY(inner)); + } + public JSAffineTransform cloneTransform() { JSAffineTransform copy = new JSAffineTransform(jsoClone(inner)); if (m4 != null) { diff --git a/Ports/LinuxPort/nativeSources/cn1_linux_gl.c b/Ports/LinuxPort/nativeSources/cn1_linux_gl.c index 60e2e57c3a8..4ed27f68e09 100644 --- a/Ports/LinuxPort/nativeSources/cn1_linux_gl.c +++ b/Ports/LinuxPort/nativeSources/cn1_linux_gl.c @@ -466,10 +466,22 @@ static void cn1GlBindDraw(CN1GlContext* c, CN1GlPipeline* p, GLuint vbo, int str glDisable(GL_BLEND); } else { glEnable(GL_BLEND); + /* The colour is weighted by the source alpha; the ALPHA channel is + * not. It accumulates coverage (a = sa + da * (1 - sa)), so a frame + * cleared opaque stays opaque whatever is blended onto it. With one + * glBlendFunc for all four channels the alpha was weighted by itself + * (sa * sa + da * (1 - sa)): a 40% black quad over an opaque clear + * left 0.76 in the FBO, cn1GlReadback handed that to cairo as the + * frame's own transparency and the form behind showed through the + * whole quad -- lighter than the background it was meant to darken. + * This is RenderState.BlendMode's contract and what the Direct3D + * backend does (cn1_windows_d3d.cpp). */ if (p->blend == 2) { - glBlendFunc(GL_SRC_ALPHA, GL_ONE); /* additive */ + glBlendFuncSeparate(GL_SRC_ALPHA, GL_ONE, + GL_ONE, GL_ONE_MINUS_SRC_ALPHA); /* additive */ } else { - glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); /* alpha */ + glBlendFuncSeparate(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA, + GL_ONE, GL_ONE_MINUS_SRC_ALPHA); /* alpha */ } } if (p->cull == 0) { diff --git a/Ports/LinuxPort/nativeSources/cn1_linux_text.c b/Ports/LinuxPort/nativeSources/cn1_linux_text.c index 1817e317cd9..18a612af368 100644 --- a/Ports/LinuxPort/nativeSources/cn1_linux_text.c +++ b/Ports/LinuxPort/nativeSources/cn1_linux_text.c @@ -234,15 +234,55 @@ JAVA_INT com_codename1_impl_linux_LinuxNative_fontHeight___long_R_int(CODENAME_O return cn1FontOrDefault(font)->height; } +/* + * The Font.STYLE_* bits a "native:" scheme name asks for: native:MainBold and + * native:MainBlack are bold, every native:Italic* name is italic, and + * native:ItalicBold is both. Any other name asks for nothing. + * + * The style of such a font is in its NAME. The portable way to ask for one is + * Font.createTrueTypeFont("native:MainBold", "native:MainBold").derive(px, + * Font.STYLE_PLAIN) -- Font.createTrueTypeFont(name, sizeMm) is exactly that -- + * so the weight passed to derive is PLAIN and only the load can honour the + * name. This port used to map every native: name to regular "Sans", and bold + * and italic text came out in the regular face wherever the caller did not + * repeat the style in derive's weight (which iOS ignores, so portable code + * does not). + * + * Kept free of Pango and of the VM's types on purpose: + * NativeFontStyleContractTest (vm/tests) compiles this function alone and + * runs it, since nothing else in CI builds this file off Linux. + */ +static int cn1LinuxNativeFontStyle(const char* name) { + int style = 0; + if (name == 0 || strncmp(name, "native:", 7) != 0) { + return 0; + } + if (strstr(name + 7, "Bold") != 0 || strstr(name + 7, "Black") != 0) { + style |= CN1_STYLE_BOLD; + } + if (strstr(name + 7, "Italic") != 0) { + style |= CN1_STYLE_ITALIC; + } + return style; +} + +/* The system UI family in the weight and slant a "native:" name asks for. + * deriveTrueTypeFont copies the description, so the style survives a derive + * to another size. */ +static CN1Font* cn1LinuxNativeSchemeFont(const char* name) { + int style = cn1LinuxNativeFontStyle(name); + return cn1MakeFont("Sans", (style & CN1_STYLE_BOLD) != 0 ? PANGO_WEIGHT_BOLD : PANGO_WEIGHT_NORMAL, + (style & CN1_STYLE_ITALIC) != 0, 18); +} + JAVA_LONG com_codename1_impl_linux_LinuxNative_loadTrueTypeFont___java_lang_String_java_lang_String_R_long(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT fontName, JAVA_OBJECT fileName) { const char* name = fontName == JAVA_NULL ? "Sans" : stringToUTF8(threadStateData, fontName); - const char* family = name; (void) fileName; - /* "native:" scheme maps to the system UI family. */ + /* "native:" scheme maps to the system UI family, styled by its name. */ if (strncmp(name, "native:", 7) == 0) { - family = "Sans"; + return (JAVA_LONG) (intptr_t) cn1LinuxNativeSchemeFont(name); } - return (JAVA_LONG) (intptr_t) cn1MakeFont(family, PANGO_WEIGHT_NORMAL, 0, 18); + return (JAVA_LONG) (intptr_t) cn1MakeFont(name, PANGO_WEIGHT_NORMAL, 0, 18); } JAVA_LONG com_codename1_impl_linux_LinuxNative_loadTrueTypeFontFromMemory___java_lang_String_byte_1ARRAY_R_long(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT fontName, JAVA_OBJECT data) { @@ -261,7 +301,7 @@ JAVA_LONG com_codename1_impl_linux_LinuxNative_loadTrueTypeFontFromMemory___java family[0] = 0; if (data == JAVA_NULL || (len = (int) (*(JAVA_ARRAY) data).length) <= 0) { if (strncmp(name, "native:", 7) == 0) { - name = "Sans"; + return (JAVA_LONG) (intptr_t) cn1LinuxNativeSchemeFont(name); } return (JAVA_LONG) (intptr_t) cn1MakeFont(name, PANGO_WEIGHT_NORMAL, 0, 18); } diff --git a/Ports/WindowsPort/nativeSources/cn1_windows_text.c b/Ports/WindowsPort/nativeSources/cn1_windows_text.c index 8bf6d5f91b5..128eaeeb2ee 100644 --- a/Ports/WindowsPort/nativeSources/cn1_windows_text.c +++ b/Ports/WindowsPort/nativeSources/cn1_windows_text.c @@ -242,6 +242,29 @@ JAVA_VOID com_codename1_impl_windows_WindowsNative_drawString___long_java_lang_S * (Segoe UI) with weight/slant from the suffix; any other name is a literal * family. */ +/* + * The Font.STYLE_* bits a "native:" scheme name asks for: native:MainBold and + * native:MainBlack are bold, every native:Italic* name is italic, and + * native:ItalicBold is both. Any other name asks for nothing. + * + * Kept free of Windows and of the VM's types on purpose: + * NativeFontStyleContractTest (vm/tests) compiles this function alone and + * runs it, since nothing else in CI builds this file off Windows. + */ +static int cn1WinNativeFontStyle(const wchar_t* name) { + int style = 0; + if (name == 0 || wcsncmp(name, L"native:", 7) != 0) { + return 0; + } + if (wcsstr(name + 7, L"Bold") != 0 || wcsstr(name + 7, L"Black") != 0) { + style |= CN1_STYLE_BOLD; + } + if (wcsstr(name + 7, L"Italic") != 0) { + style |= CN1_STYLE_ITALIC; + } + return style; +} + /* * Fallback when no bundled TTF was registered: resolve a system family. The * "native:" scheme names (native:MainLight, native:ItalicBold, ...) map to the @@ -252,13 +275,7 @@ static CN1Font* cn1WinSystemFontForName(const WCHAR* name, float dpi) { const wchar_t* family = L"Segoe UI"; int style = 0; if (name != NULL && wcsncmp(name, L"native:", 7) == 0) { - const wchar_t* suffix = name + 7; - if (wcsstr(suffix, L"Bold") != NULL || wcsstr(suffix, L"Black") != NULL) { - style |= CN1_STYLE_BOLD; - } - if (wcsstr(suffix, L"Italic") != NULL) { - style |= CN1_STYLE_ITALIC; - } + style = cn1WinNativeFontStyle(name); } else if (name != NULL && name[0] != L'\0') { family = name; } @@ -324,13 +341,25 @@ JAVA_LONG com_codename1_impl_windows_WindowsNative_loadTrueTypeFontFromMemory___ /* * deriveTrueTypeFont produces a new peer at the requested pixel size and weight, * reusing the base font's family. weight carries the CN1 STYLE_* bits. + * + * The base font's own style is kept and the requested bits are added to it. A + * "native:" scheme font has its weight and slant in its NAME, which the load + * above turned into base->style; the portable request is + * Font.createTrueTypeFont("native:MainBold", "native:MainBold").derive(px, + * Font.STYLE_PLAIN) -- Font.createTrueTypeFont(name, sizeMm) is exactly that -- + * so the weight arriving here is PLAIN. Building the derived format from the + * weight alone threw the name's style away, and every native:*Bold and + * native:Italic* font was drawn in the regular face at any size but the + * loaded one. The Linux port's derive copies the base description for the + * same reason. A bundled TrueType file loads with style 0, so it is unchanged. */ JAVA_LONG com_codename1_impl_windows_WindowsNative_deriveTrueTypeFont___long_float_int_R_long( CODENAME_ONE_THREAD_STATE, JAVA_LONG __cn1Arg1, JAVA_FLOAT __cn1Arg2, JAVA_INT __cn1Arg3) { CN1Font* base = (CN1Font*) (intptr_t) __cn1Arg1; const wchar_t* family = (base != NULL && base->family != NULL) ? base->family : L"Segoe UI"; float px = __cn1Arg2 > 0.0f ? __cn1Arg2 : 15.0f; - CN1Font* font = cn1WinMakeFont(family, px, 0, __cn1Arg3); + int style = (base != NULL ? base->style : 0) | __cn1Arg3; + CN1Font* font = cn1WinMakeFont(family, px, 0, style); return (JAVA_LONG) (intptr_t) font; } diff --git a/Ports/iOSPort/nativeSources/CN1GL3D.m b/Ports/iOSPort/nativeSources/CN1GL3D.m index 63e7b4d38b4..6abe9dd5d62 100644 --- a/Ports/iOSPort/nativeSources/CN1GL3D.m +++ b/Ports/iOSPort/nativeSources/CN1GL3D.m @@ -262,7 +262,12 @@ - (CN1GL3DPipeline *)pipelineForKey:(NSString *)key source:(NSString *)mslSource desc.colorAttachments[0].rgbBlendOperation = MTLBlendOperationAdd; desc.colorAttachments[0].alphaBlendOperation = MTLBlendOperationAdd; desc.colorAttachments[0].sourceRGBBlendFactor = MTLBlendFactorSourceAlpha; - desc.colorAttachments[0].sourceAlphaBlendFactor = MTLBlendFactorSourceAlpha; + // One, not SourceAlpha: the alpha channel accumulates coverage + // (a = sa + da * (1 - sa)) so a frame cleared opaque stays opaque. + // Weighted by itself it left 0.76 under a 40% quad, which the opaque + // layer hides here but which made the same scene translucent on the + // ports that read the frame back (RenderState.BlendMode's contract). + desc.colorAttachments[0].sourceAlphaBlendFactor = MTLBlendFactorOne; desc.colorAttachments[0].destinationRGBBlendFactor = MTLBlendFactorOneMinusSourceAlpha; desc.colorAttachments[0].destinationAlphaBlendFactor = MTLBlendFactorOneMinusSourceAlpha; } else if (blendMode == 2) { // ADDITIVE @@ -272,7 +277,8 @@ - (CN1GL3DPipeline *)pipelineForKey:(NSString *)key source:(NSString *)mslSource desc.colorAttachments[0].sourceRGBBlendFactor = MTLBlendFactorSourceAlpha; desc.colorAttachments[0].sourceAlphaBlendFactor = MTLBlendFactorOne; desc.colorAttachments[0].destinationRGBBlendFactor = MTLBlendFactorOne; - desc.colorAttachments[0].destinationAlphaBlendFactor = MTLBlendFactorOne; + // Coverage, as in the ALPHA mode above and on every other backend. + desc.colorAttachments[0].destinationAlphaBlendFactor = MTLBlendFactorOneMinusSourceAlpha; } else { desc.colorAttachments[0].blendingEnabled = NO; } diff --git a/docs/demos/common/src/main/snippets/developer-guide/unity-interop.cs b/docs/demos/common/src/main/snippets/developer-guide/unity-interop.cs new file mode 100644 index 00000000000..3a376e8ab78 --- /dev/null +++ b/docs/demos/common/src/main/snippets/developer-guide/unity-interop.cs @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +// Developer guide snippets for the Unity compatibility chapter. + +// tag::unity-interop-csharp-001[] +using System; +using System.Collections.Generic; +using UnityEngine; + +public class Board : MonoBehaviour +{ + public GameObject tilePrefab; + + private int[,] cells; + private readonly Dictionary tiles = new Dictionary(); + + void Start() + { + TextAsset level = Resources.Load("level1"); + string[] rows = level.text.Split(new[] { '\n' }, StringSplitOptions.RemoveEmptyEntries); + cells = new int[rows.Length, rows[0].Split(',').Length]; + for (int r = 0; r < cells.GetLength(0); r++) + { + string[] columns = rows[r].Split(','); + for (int c = 0; c < cells.GetLength(1); c++) + { + int.TryParse(columns[c], out cells[r, c]); + if (cells[r, c] != 0) + { + GameObject tile = Instantiate(tilePrefab, new Vector3(c, -r, 0), Quaternion.identity); + tiles[tile] = new Vector2(c, r); + } + } + } + PlayerPrefs.SetInt("lastLevel", 1); + } +} +// end::unity-interop-csharp-001[] + +// tag::unity-interop-csharp-002[] +// What the game asks of the application around it. The game compiles +// against this interface alone; the application supplies the object. +public interface IPlatformServices +{ + void SignIn(); + void Share(string text); + void ShowLeaderboard(int score); +} + +public static class Platform +{ + public static IPlatformServices Services; +} +// end::unity-interop-csharp-002[] + +// tag::unity-interop-csharp-003[] +public class Score : MonoBehaviour +{ + public int points; + + public int Best { get; private set; } + + public void Add(int amount) + { + points += amount; + if (points > Best) + { + Best = points; + } + } + + public Vector2 Where() + { + return transform.position; + } + + private void Start() + { + Platform.Services?.SignIn(); + } + + private void OnApplicationPause(bool paused) + { + Debug.Log("score paused=" + paused); + } + + private void Update() + { + if (Platform.Services == null) + { + return; + } + if (Input.GetKeyDown(KeyCode.L)) + { + Platform.Services.ShowLeaderboard(Best); + } + if (Input.GetKeyDown(KeyCode.S)) + { + Platform.Services.Share("I scored " + Best); + } + } +} +// end::unity-interop-csharp-003[] diff --git a/docs/demos/common/src/main/snippets/developer-guide/unity-interop.sh b/docs/demos/common/src/main/snippets/developer-guide/unity-interop.sh new file mode 100644 index 00000000000..5ea9ffb8a68 --- /dev/null +++ b/docs/demos/common/src/main/snippets/developer-guide/unity-interop.sh @@ -0,0 +1,40 @@ +// Generated from docs/developer-guide source blocks. Edit the guide snippets here, not inline. + +// tag::unity-interop-bash-001[] +mvn cn1:import-unity-project -Dsource=/path/to/MyUnityGame +// end::unity-interop-bash-001[] + +// tag::unity-interop-bash-002[] +mvn cn1:run +mvn cn1:build -Dcodename1.platform=ios +// end::unity-interop-bash-002[] + +// tag::unity-interop-bash-003[] +dotnet --list-sdks +mvn package -Dcn1.unity.dotnet=/opt/dotnet/dotnet +// end::unity-interop-bash-003[] + +// tag::unity-interop-bash-004[] +./gradlew run +./gradlew buildIos -Pcn1.unity.dotnet=/opt/dotnet/dotnet +// end::unity-interop-bash-004[] + +// tag::unity-interop-bash-005[] +mvn package +// end::unity-interop-bash-005[] + +// tag::unity-interop-bash-006[] +mvn cn1:build -Dcodename1.platform=android +mvn cn1:build -Dcodename1.platform=ios +mvn cn1:build -Dcodename1.platform=javascript +// end::unity-interop-bash-006[] + +// tag::unity-interop-bash-007[] +mvn package -Dcn1.unity.skip=true +// end::unity-interop-bash-007[] + +// tag::unity-interop-bash-008[] +less common/target/unity/dotnet.log +less common/target/unity/translate.log +less common/target/unity/scene-compiler.log +// end::unity-interop-bash-008[] diff --git a/docs/demos/common/src/main/snippets/developer-guide/unity-interop.txt b/docs/demos/common/src/main/snippets/developer-guide/unity-interop.txt new file mode 100644 index 00000000000..aa8dd7eb236 --- /dev/null +++ b/docs/demos/common/src/main/snippets/developer-guide/unity-interop.txt @@ -0,0 +1,128 @@ +// tag::unityInteropMainClass[] +package com.example.mygame; + +public class MyGame extends com.codename1.unitycompat.app.UnityApplication { + @Override + protected void installProject() { + com.codename1.generated.unity.UnityAppImpl.install(); + } +} +// end::unityInteropMainClass[] + +// tag::unityInteropApplication[] +public class MyGame extends UnityApplication { + private final GameServices services = new GameServices(this); + private final Label scoreLabel = new Label("Score 0"); + + @Override + protected void installProject() { + UnityAppImpl.install(); + } + + @Override + protected void onProjectInstalled() { + // No scene is built yet: every Awake and Start finds this set. + Platform.Services = services; + } + + @Override + protected Form createForm(UnityGameView gameView) { + Form form = new Form(new BorderLayout()); + Toolbar toolbar = new Toolbar(); + form.setToolbar(toolbar); + form.setTitle("My Game"); + form.setScrollable(false); + toolbar.addMaterialCommandToRightBar("", FontImage.MATERIAL_LEADERBOARD, + e -> callInFrame(() -> services.ShowLeaderboard(score().get_Best()))); + toolbar.addMaterialCommandToSideMenu("Restart", FontImage.MATERIAL_REPLAY, + e -> callInFrame(() -> score().points = 0)); + form.add(BorderLayout.CENTER, gameView); + form.add(BorderLayout.SOUTH, scoreLabel); + return form; + } + + @Override + protected void onStarted(Form gameForm, UnityGameView gameView) { + // Twice a second: read the score inside the frame, show it on the + // event dispatch thread. + UITimer.timer(500, true, gameForm, () -> callInFrame(() -> { + int points = score().points; + CN.callSerially(() -> { + scoreLabel.setText("Score " + points); + gameForm.revalidate(); + }); + })); + } + + // Only ever called inside the frame. + private static Score score() { + return (Score) GameObject.Find("Ball").GetComponent(Score.class); + } +} +// end::unityInteropApplication[] + +// tag::unityInteropServices[] +final class GameServices implements IPlatformServices { + private final UnityApplication app; + + GameServices(UnityApplication app) { + this.app = app; + } + + // Scripts call these inside a frame, which isn't always on the event + // dispatch thread. Each one hands its work to that thread. + + @Override + public void SignIn() { + CN.callSerially(() -> { + if (Preferences.get("player", null) != null) { + return; + } + GoogleConnect.getInstance().signIn( + "YOUR_CLIENT_ID.apps.googleusercontent.com", + "com.example.mygame:/oauth2redirect", + "openid", "email" + ).ready(tokens -> Preferences.set("player", tokens.getEmail())); + }); + } + + @Override + public void Share(String text) { + CN.callSerially(() -> CN.share(text, null, null)); + } + + @Override + public void ShowLeaderboard(int score) { + CN.callSerially(() -> showLeaderboard(score)); + } + + private void showLeaderboard(int score) { + app.getView().pause(); + Form board = new Form(BoxLayout.y()); + Toolbar toolbar = new Toolbar(); + board.setToolbar(toolbar); + board.setTitle("Leaderboard"); + board.add(new Label("Your best: " + score)); + toolbar.setBackCommand("", e -> { + app.getForm().showBack(); + app.getView().resume(); + }); + board.show(); + } +} +// end::unityInteropServices[] + + // tag::unityInteropCallingRules[] + // Inside a frame: in a Runnable given to callInFrame, or in a + // method a script called. + // By the name of its object, then by the script's class. + Score score = (Score) GameObject.Find("Ball").GetComponent(Score.class); + // Or the first one in the scene, whatever object it is on. + Score first = (Score) com.codename1.unitycompat.unityengine.Object + .FindObjectOfType(Score.class); + + int points = score.points; // public int points; + int best = score.get_Best(); // public int Best { get; } + Vector2 where = score.Where(new Vector2()); // public Vector2 Where() + float height = where.y; + // end::unityInteropCallingRules[] diff --git a/docs/developer-guide/Unity-Interop.asciidoc b/docs/developer-guide/Unity-Interop.asciidoc new file mode 100644 index 00000000000..7ce2ce3bf11 --- /dev/null +++ b/docs/developer-guide/Unity-Interop.asciidoc @@ -0,0 +1,1240 @@ +== Running Unity 2D games + +A Unity 2D project -- C# scripts, scenes, prefabs, sprites and project +settings -- builds and runs as a Codename One application without changes to +its scripts or its assets. You copy the project into a Codename One project, +the build compiles the C# and translates it to Java bytecode, and the game +runs on every Codename One target: iOS, Android, the JavaScript port, the +desktop ports and the simulator. + +image::img/unity-interop-fruitopia.png["Fruitopia, an open source Unity platformer, running on the Codename One runtime: the frog hero in mid-jump beside a spiked ball, over a tile map level with a fruit counter",scaledwidth=70%] + +No Unity player and no .NET runtime ship with the application. Scripts become +ordinary classes, scenes and prefabs become generated code that creates their +objects, and a runtime that implements the `UnityEngine` API draws the game +into a Codename One component. + +That component is the point. The game sits on a Codename One form, inside a +Codename One application, so the same project can put a toolbar and a side +menu around it, show a settings screen or a store, and use sign-in, in-app +purchase, push, advertising and sharing from one Java codebase on every +target. Scripts and Java call each other; +<> shows how. + +The scope is 2D games built from sprites, tile maps, 2D physics, animation +controllers, particle effects, a canvas interface and scripts. Five open +source games run unmodified today, among them the platformer in the image +above; see <>. Read +<> before you plan a port. + +=== Requirements + +Besides what every Codename One project needs, a project with Unity sources +needs the https://dotnet.microsoft.com/download[.NET SDK], version 6 or newer, +on the machine that runs the build. The build uses its C# compiler and nothing +else: no package is downloaded and no Unity installation or Unity license is +needed. + +The build looks for the `dotnet` command in this order: + +. The `cn1.unity.dotnet` property, which names the command or its directory. +. The `DOTNET_ROOT` environment variable. +. The `PATH`. + +[source,bash] +---- +include::../demos/common/src/main/snippets/developer-guide/unity-interop.sh[tag=unity-interop-bash-003,indent=0] +---- + +A project with no Unity sources doesn't need the SDK. An unchanged Unity +project isn't compiled again, so only the builds that follow a change to a +script or an asset use it. + +The Unity project must store its assets as text, which is the default in +current Unity versions (*Edit > Project Settings > Editor > Asset +Serialization > Force Text*). The +build reads scenes, prefabs and `.meta` files as the YAML text Unity writes in +that mode. + +=== Quick start + +Create a Codename One application project (for example from +https://start.codenameone.com[the initializr]), then import the Unity project +into it: + +[source,bash] +---- +include::../demos/common/src/main/snippets/developer-guide/unity-interop.sh[tag=unity-interop-bash-001,indent=0] +---- + +Then run and build as usual: + +[source,bash] +---- +include::../demos/common/src/main/snippets/developer-guide/unity-interop.sh[tag=unity-interop-bash-002,indent=0] +---- + +The next section walks through the same steps and what each one prints. + +[[unity-interop-porting]] +=== Porting a project, step by step + +==== 1. Import the project + +Run `cn1:import-unity-project` from the root of the Codename One project and +point `-Dsource` at the directory that holds `Assets` and `ProjectSettings`. +The goal does four things: + +* Copies `Assets` and `ProjectSettings` into `common/src/main/unity`. It + leaves out what the Unity editor derives from them (`Library`, `Temp`, + `obj`) and the version control directories. +* Adds the `codenameone-unity-compat` runtime and the `compile-unity` goal to + the `common` module's `pom.xml`. +* Rewrites the Codename One main class so that it starts the game. The + previous class is kept beside it with a `.pre-unity-import` suffix. +* Prints what it copied, and lists every file the build won't use. A + compiled plugin shows up here as `not built: Assets/Plugins/Foo.dll + (compiled code: only C# source is translated)`. + +The copy mirrors the Unity project. Keep editing the game in Unity and run +the import again: it replaces the files it copied, removes the ones the Unity +project no longer has, and leaves alone both the files you added by hand and +a main class you changed since the last import. + +You can also copy the files by hand. The directory mirrors the root of a Unity +project: + +---- +common/src/main/unity/ + Assets/ scripts, scenes, prefabs, sprites and their .meta files + ProjectSettings/ the input axes, tags, layers and physics settings +---- + +Keep the `.meta` files. Unity identifies an asset by the identifier in its +`.meta` file, and that's how a scene finds the sprite or the script it refers +to. + +==== 2. Run the first build + +Build the project. This is the step that needs the .NET SDK: + +[source,bash] +---- +include::../demos/common/src/main/snippets/developer-guide/unity-interop.sh[tag=unity-interop-bash-005,indent=0] +---- + +The build compiles the scripts, translates them and compiles the scenes. A +project inside the supported surface ends with the line `Compiled the Unity +project`. A script that uses an API outside the surface stops the build here +with the script file, the line and the name of the missing member, so the list of +changes a port needs is in front of you after the first build rather than +after the first crash. <> shows +both forms of that message. + +==== 3. Read the build report + +The translator and the scene compiler report everything they left out or +approximated. Maven and Gradle print each line in the build output, prefixed +`src/main/unity:`: + +* A *warning* names something that changes how the game behaves or looks: a + component the runtime doesn't have, a perspective camera made orthographic, + a sprite a scene refers to that isn't under `Assets`, a script method Unity + would call and the runtime doesn't. The build log shows + these at warning level, and the last line counts them: `Compiled the Unity + project with 2 warning(s): what they name will not behave as it does in + Unity`. +* A *note* names something left out that doesn't change the game, or a + substitution you might want to know about. Maven shows notes at info level. + +A line about a scene names the scene or prefab file, the GameObject and the +component's file ID, the number after the `&` in the scene's YAML. A warning +about a script names the class, the method, the file and the line. This is the +complete report of the Sokoban game from +<>, seven notes and no warnings, shown +without the prefix: + +---- +Sokoban.unity: the FlareLayer component of GameObject 'Camera with script' (component file ID 1187623029) was left out; nothing in a 2D scene is drawn through it +Sokoban.unity: the GUILayer component of GameObject 'Camera with script' (component file ID 1187623030) was left out; nothing in a 2D scene is drawn through it +Sokoban.unity: the StandaloneInputModule of GameObject 'EventSystem' (component file ID 1460168504) was left out; the event system beside it routes the pointer to the controls itself, and nothing moves between controls by keyboard +Hero.png is a polygon sprite: it was cut to its outline when the project was built, into unity-polygon-Hero.png (256 by 256 pixels), and is drawn as that image +Ball.png is a polygon sprite: it was cut to its outline when the project was built, into unity-polygon-Ball.png (256 by 256 pixels), and is drawn as that image +Sokoban.unity: the image of GameObject 'RestartButton' (file ID 1532672074) uses Unity's built-in interface sprite 10905, which is not in the project; a plain rectangle of the image's colour is drawn instead +the input settings define joystick axes; those read as zero, and the keys of the same axes work +---- + +The full output of each tool is in `common/target/unity`: + +[source,bash] +---- +include::../demos/common/src/main/snippets/developer-guide/unity-interop.sh[tag=unity-interop-bash-008,indent=0] +---- + +==== 4. Run in the simulator + +`mvn cn1:run` opens the game in the Codename One simulator. It shows the +first scene of the Unity build settings in a form that fills the screen. If +the build settings list no scene, every scene under `Assets` is compiled in +order of path and the first one starts. + +The simulator has a keyboard, so a game written for the desktop plays as it +is. Keys reach `Input` under Unity's key codes, the arrow keys drive the +default `Horizontal` and `Vertical` axes beside WASD, and the pointer is the +mouse. + +==== 5. Plan the input for a phone + +Touches reach a script two ways. `Input.touchCount`, `Input.touches` and +`Input.GetTouch` report every finger on the screen with its phase, so a game +written for touch plays as it is. The first finger also reads as mouse +button 0, with `Input.mousePosition` at the touch, and canvas buttons respond +to it. A game played with the mouse or with canvas buttons therefore needs +nothing more on a phone. + +A script that shows its touch controls only on a handheld can keep asking +`Application.isMobilePlatform`, which is true on iOS and on Android. In a +browser, ask `Input.touchSupported` instead: it reports whether the device has +a touch screen, which is the question on a phone's browser and a laptop's +alike. + +A game played with keys needs a way to press them on a device that has none. +The game's view accepts key events from Java code: call `keyPressed` and +`keyReleased` on it with the character of the key, and the script sees +`Input.GetKey` exactly as if a keyboard had sent it. Add Codename One buttons +around the game for that, as +<> +describes. + +==== 6. Build for a device + +Device builds are ordinary Codename One builds. Nothing about them is +specific to Unity sources: + +[source,bash] +---- +include::../demos/common/src/main/snippets/developer-guide/unity-interop.sh[tag=unity-interop-bash-006,indent=0] +---- + +==== 7. Iterate + +After a change to a script, a scene or an asset, build again. The build +fingerprints `Assets` and `ProjectSettings` and repeats the C# compile, the +translation and the scene compile only when one of those files changed. A +change to your Java code alone doesn't run the .NET SDK. + +To leave the Unity project out of one build, set `cn1.unity.skip`. The classes +an earlier build produced stay in place: + +[source,bash] +---- +include::../demos/common/src/main/snippets/developer-guide/unity-interop.sh[tag=unity-interop-bash-007,indent=0] +---- + +[[unity-interop-codename-one]] +=== Using Codename One from your game + +The game isn't the whole application. It's one component, a `UnityGameView`, +on a Codename One form, in an application whose main class is Java. The rest +of this guide therefore applies to the same project and reaches every target +the game does: the user interface toolkit and its themes, sign-in, in-app +purchase, push, advertising, sharing, storage, and networking. + +Code travels in three directions, and this section takes them in turn: + +* Codename One around the game: forms, a toolbar, a side menu and other + screens, written in Java. +* Java calling the game: reading a score, or restarting a level. +* The game calling Codename One: a script that asks to sign in, to share a + result or to open a leaderboard. + +The samples are one small application. Its Unity project has two scripts +besides the game's own, and its Java side is two classes. + +[[unity-interop-entry-point]] +==== The entry point + +The import writes a main class that extends +`com.codename1.unitycompat.app.UnityApplication`. A project that has no main +class of its own gets the same class generated by the build. This is the whole +class, for an application whose main class is `com.example.mygame.MyGame`: + +[source] +---- +include::../demos/common/src/main/snippets/developer-guide/unity-interop.txt[tag=unityInteropMainClass,indent=0] +---- + +`installProject` is the one method you must have. It names the class the +scene compiler generated, in source, because a device build has no reflection +to find it by name. `UnityApplication` gives you more places to work from, in +the order they're called: + +* `onProjectInstalled()` runs once the project is installed and before the + first scene is built, so before the first `Awake` of any script. +* `createForm(UnityGameView gameView)` returns the form the game is shown in. +* `onStarted(Form gameForm, UnityGameView gameView)` runs once, after the + first scene is loaded and the game loop runs. +* `getForm()` and `getView()` return the same two objects later. +* `callInFrame(Runnable)` is static, and runs your code inside the game's + frame. <> explains + when you need it. + +==== Codename One screens around the game + +Override `createForm` to decide what surrounds the game. The default returns a +form with a `BorderLayout`, a hidden title area and the view in the center. +Return your own form instead; the one rule is that it must contain the view. +This main class gives the game a title bar with a leaderboard command, a side +menu with a Restart command and a score label under the playfield: + +[source] +---- +include::../demos/common/src/main/snippets/developer-guide/unity-interop.txt[tag=unityInteropApplication,indent=0] +---- + +`UnityGameView` is in `com.codename1.unitycompat.unityengine.ui`. It extends +the `GameView` of the `com.codename1.gaming` package, so it sizes and lays out +like any other component. Here it takes the center of a `BorderLayout`, and +the game draws into whatever size the layout gives the view. + +Everything around the view is ordinary Codename One user interface. The +toolbar, the side menu and the label take their look from the application's +theme, so you style them in `theme.css` as +<> describes, and +<<_toolbar,Toolbar>> covers the commands, the search field and the side menu. +The theme doesn't reach inside the view: what the game draws is the game's +own. + +==== Another form, and back to the game + +A settings screen, a store or a leaderboard is a second form. Pause the view +before you show it and resume the view when the player returns. A paused view +runs no script and advances no game time; it keeps its last frame. Scripts +that declare `OnApplicationPause` or `OnApplicationFocus` are told of both: + +[source] +---- +include::../demos/common/src/main/snippets/developer-guide/unity-interop.txt[tag=unityInteropServices,indent=0] +---- + +`showLeaderboard` at the end of the class is the part that matters here: it +pauses the view, shows a form with a back command, and the back command shows +the game's form again and resumes. + +`UnityApplication` already pauses the game when the application goes to the +background and resumes it on return, so no frame is simulated while nobody +sees it and `Time.time` doesn't jump. A script that calls `Application.Quit()` +exits the application. + +==== Stepping the game by hand + +A test that captures a scene, or a tool that renders one, needs the same +picture on a fast device and on a slow one. The view normally simulates the +time that passed since its last frame, so the picture depends on how fast the +frames came. `gameView.holdClock()` takes the clock away from the game: a +frame then simulates nothing and draws the scene as it stands. +`gameView.advance(90, 1f / 60)` asks for 90 steps of a sixtieth of a second +each, and the next frame the view draws runs every one, however long that +frame took. `framesAdvanced()` grows by the number of steps once they've been +run and the scene shows the last of them, which is the moment to capture. +`releaseClock()` hands the clock back. + +A view whose clock is held takes no input: keys and touches that arrive are +dropped, so nothing but the steps you ask for moves the scene. Calls queued +with `callInFrame` still run. + +To show one scene of a project by itself, start the runtime with +`UnityRuntime.begin(index)` where `UnityApplication` calls `begin()`. The index is the +scene's place in the build settings. `UnityRuntime.reset()` takes everything +down again, the objects, the clock and the application's settings, so the next +scene starts from nothing. Call `Random.InitState` before `begin` when the +scene has particle systems or scripts that ask for random numbers. + +==== On-screen controls + +For a game that reads keys, add buttons in `createForm` and forward their +press and release to the view. `gameView.keyPressed('a')` followed by +`gameView.keyReleased('a')` is one tap of the A key, and the script's +`Input.GetKey(KeyCode.A)` is true from one call to the other. Letters, digits and the space +character are passed as themselves. For the arrow keys pass the key code of +the matching game action, which `Display.getInstance().getKeyCode(Display.GAME_LEFT)` +returns. The view queues these calls and hands them to the game at the start +of its next frame, so a button's listener makes them directly. + +[[unity-interop-java-to-game]] +==== Calling the game from Java + +A translated script is an ordinary class, and Java code in the same module +calls it by name. A script with no namespace is in the package `global`; a +script in a namespace is in the package of that name. This is the script the +samples read: + +[source,csharp] +---- +include::../demos/common/src/main/snippets/developer-guide/unity-interop.cs[tag=unity-interop-csharp-003,indent=0] +---- + +And this is Java finding it and reading it, in code that runs inside a frame: + +[source] +---- +include::../demos/common/src/main/snippets/developer-guide/unity-interop.txt[tag=unityInteropCallingRules,indent=0] +---- + +The rules a Java author needs: + +* *Finding an object.* `GameObject.Find` takes the object's name and returns + `null` when the scene has none. `GetComponent` and `FindObjectOfType` take + the script's class where C# takes a type argument, and return an `Object` + to cast. The engine's classes are in + `com.codename1.unitycompat.unityengine`; its own `Object` class shares a + name with `java.lang.Object`, so write that one in full. +* *When.* A scene's objects exist once the scene is loaded, so from + `onStarted` on. A scene load replaces them, apart from the ones a script + kept with `DontDestroyOnLoad`, so find an object again after one. +* *Fields and methods* keep their C# names, capitals included. +* *Properties* are a pair of methods: `Best` is `get_Best()` and + `set_Best(int)`. +* *Types.* `int`, `float` and `bool` are `int`, `float` and `boolean`. A + `string` is a `java.lang.String`. An enumeration is an `int`. A C# array of + those is the Java array. A `List` is a + `com.codename1.unitycompat.system.collections.generic.List_1`, without its + type argument. +* *Value types.* `Vector2`, `Vector3`, `Color` and the other `UnityEngine` + value types are classes of the Java package `UnityEngine`, with their fields + public. A method that returns one takes an extra last argument, the object + the result is written into: `Where()` in C# is `Where(new Vector2())` in + Java. +* *Access.* The translated members are all public. A member the script + declared private isn't part of what the script offers; leave it alone. + +One rule covers threads. Scripts run inside the game's frame, on the thread +that draws it, and on some ports that isn't the event dispatch thread your +listeners run on. So: + +* Touch game objects and scripts inside `callInFrame`. +* Touch Codename One components on the event dispatch thread. + +`callInFrame(Runnable)` runs the code at the start of the next frame, before +any script. It's a static method of `UnityApplication`, and the view has the +same method. Calls run in the order they were made, and they run while the +view is paused too. The main class above uses it three times. The Restart +command sets a field of the script. The leaderboard command reads the best +score and hands it on. The timer reads the points inside the frame and then +passes the number to `CN.callSerially`, which sets the label on the event +dispatch thread. + +Code that a script calls is already inside the frame, and may use any script +directly. + +[[unity-interop-game-to-java]] +==== Calling Codename One from the game + +A script compiles against the `UnityEngine` API and the .NET base library, so +it can't name a Java class. It can name an interface of its own. Declare, in +the Unity project, what the game needs from the application, and a static +field to hold the object that provides it: + +[source,csharp] +---- +include::../demos/common/src/main/snippets/developer-guide/unity-interop.cs[tag=unity-interop-csharp-002,indent=0] +---- + +The interface is translated with the scripts and becomes the Java interface +`global.IPlatformServices`. Implement it in Java, with the whole Codename One +API at hand. `GameServices`, shown earlier, is that implementation: `SignIn` +uses `GoogleConnect`, `Share` opens the platform's share sheet through +`CN.share` and `ShowLeaderboard` shows a form. + +Three things make the bridge work: + +* *Register it before the first scene loads.* The main class assigns + `Platform.Services` in `onProjectInstalled`. Every `Awake` and `Start` then + finds it set; `Score` calls `SignIn()` from its `Start`. +* *Hand the work to the event dispatch thread.* A script calls the interface + from inside a frame. Each method wraps what it does in `CN.callSerially`, + and returns at once so the frame isn't held up. +* *Keep the interface to simple types.* `int`, `float`, `bool` and `string` + cross unchanged. A method that returns a value answers on the game's + thread: return something the application already holds, and don't touch a + component there. + +The script side stays plain C#. `Platform.Services?.Share("I scored " + Best)` +does nothing when no object is registered, so the scripts don't depend on the +application being there. + +From the Java side of that interface the whole platform is in reach: + +* *Sign-in* with Google, Apple, Facebook, Microsoft or any OpenID Connect + provider: <>. +* *In-app purchase and subscriptions* through `Purchase`: + <<_in_app_purchase,In-app purchase>>. +* *Advertising*, including rewarded and interstitial formats between levels: + <<_advertising,Advertising>>. +* *Push notifications*: <<_push_notifications,Push notifications>>. +* *Sharing* a score or a screenshot: `CN.share`, or the + <> component. +* *Storage and networking* for saved games and online scores: `Preferences`, + <<_storage,Storage>> and + <<_network_manager_connection_request,`ConnectionRequest`>>. +* *Anything a platform SDK offers* that Codename One doesn't wrap: + <<_native_interfaces,native interfaces>>. + +`PlayerPrefs` needs no bridge. The runtime keeps it in Codename One's +`Preferences` storage, so a high score outlives the application on every +target. + +=== How the build works + +A single step runs as part of the normal build of the `common` module: +*`compile-unity`*, in `generate-sources`. It does three things: + +. The .NET SDK compiles every `.cs` file under `Assets` to one assembly, + with the language level set to C# 9. It leaves out the scripts in an `Editor` directory, as Unity leaves them + out of a player build. The scripts compile against a `UnityEngine` assembly + that ships with the runtime, so a script that uses a `UnityEngine` member + the runtime doesn't have fails here, with the C# compiler's own file and + line. +. The translator reads the assembly and writes Java class files, one class + for each C# type. A type in a namespace lands in the package of the same + name. A type with no namespace, which is what most Unity scripts are, lands + in the package `global`, because a Java class in the default package can't + be imported. The translator also checks every call into the .NET base + library against the runtime and lists the members the runtime lacks. +. The scene compiler turns every scene and prefab into Java source: a + generated class, `com.codename1.generated.unity.UnityAppImpl`, that creates + the objects with `new`, assigns the values the Inspector stored and connects + the references. It copies the sprites, sounds and text assets next to it. + +Nothing is created by reflection and no scene file is parsed on the device. +That's what lets the same code run on iOS and in the browser, where the +translated application keeps only the code something refers to. Three things +Unity resolves by name at run time are resolved by the build instead: + +* `Invoke("Fire", 1f)` and `InvokeRepeating` call a dispatcher the translator + writes into each script, which maps the method names of that script to + calls. +* A button's persistent `onClick` calls, the ones set up in the Inspector, + become direct calls in the generated scene code. +* `Resources.Load("level1")` is a lookup the scene compiler writes from the + files it found under the `Resources` folders. The asset is already part of + the application when a script asks for it. + +Two representations are worth knowing as a script author: + +* *Value types keep C#'s copy semantics.* `Vector2`, `Vector3`, `Quaternion`, + `Color`, `Bounds` and your own structs are copied on assignment and when + passed, exactly as C# specifies. Arithmetic on them writes into storage the + caller owns instead of allocating an object for each result, so vector math + in `Update` doesn't produce garbage. +* *A rectangular array is one flat array.* `int[,]` and arrays of higher rank + are stored as a single array with every index checked against its own + dimension. A tile grid costs one allocation and each access is one multiply + and one add. + +The `codenameone-unity-compat` dependency is added to the `common` module by +the project's `unity-compat` profile, which activates when `src/main/unity` +exists. + +=== Gradle projects + +A Gradle project builds a Unity project the same way. The plugin sees +`src/main/unity`, adds the `codenameone-unity-compat` runtime to the +application and runs a `compileUnity` task before the Java sources compile, so +there is no dependency to declare and no task to call. A Maven project that +has a Unity project keeps it when you convert it with `cn1:convert-to-gradle`. + +The import is a Maven goal. In a project that was created for Gradle, copy +`Assets` and `ProjectSettings` into `src/main/unity` by hand. Then either +delete the main class the project came with, and the build generates one that +starts the game, or change it as <> +describes. + +Name the .NET SDK with `-Pcn1.unity.dotnet` when it isn't on the `PATH`: + +[source,bash] +---- +include::../demos/common/src/main/snippets/developer-guide/unity-interop.sh[tag=unity-interop-bash-004,indent=0] +---- + +Gradle skips `compileUnity` as up to date while the Unity project is +unchanged. To leave the Unity project out of one build, exclude the task with +`-x compileUnity`; the classes an earlier build produced stay in place. The +work files and the logs of the C# compiler, the translator and the scene +compiler are in `build/cn1-unity/state/unity`. + +[[unity-interop-supported]] +=== What a project can use + +This is the surface the five games in the next sections were built on. It +covers what a 2D game with sprites, tile maps, physics, animation and a +canvas interface is made of. + +==== C# and the base library + +* The language as Unity projects use it: classes, structs, interfaces, enums, + generics, delegates and events, lambdas and closures, iterators, `foreach`, + `ref` and `out` parameters, `switch` on strings and switch expressions, + tuples, string interpolation, exceptions, properties, indexers and operator + overloading. +* Strings: concatenation, `Format` with plain `{0}` items, `Join`, `Split` + on characters or on strings with or without `StringSplitOptions`, + `Substring`, `IndexOf`, `Contains`, `StartsWith`, `EndsWith`, `Replace`, + `Trim`, and the invariant case conversions. +* Numbers: `int.Parse`, `int.TryParse`, `System.Math` and `Mathf`. +* `List` and `Dictionary`. A dictionary enumerates in insertion + order and has `TryGetValue`, `ContainsKey`, `Remove`, `Keys`, `Values` and + `foreach` over its pairs. +* Arrays, including rectangular arrays of any rank with `GetLength`. +* The `Action` and `Func` delegates and these LINQ operators: `Where`, + `Select`, `Any`, `All`, `Count`, `Contains`, `First`, `FirstOrDefault`, + `Last`, `LastOrDefault`, `ElementAt`, `Take`, `Skip`, `Distinct`, `Concat`, + `Reverse`, `ToList` and `ToArray`. + +The script below uses several of these together: a level read from a text +asset in a `Resources` folder into a two-dimensional array, with the tiles it +creates tracked in a dictionary. It builds as written. + +[source,csharp] +---- +include::../demos/common/src/main/snippets/developer-guide/unity-interop.cs[tag=unity-interop-csharp-001,indent=0] +---- + +==== Scripts and objects + +* `MonoBehaviour` with these messages: `Awake`, `OnEnable`, `Start`, + `FixedUpdate`, `Update`, `LateUpdate`, `OnDisable`, `OnDestroy`, and the 2D + collision and trigger messages (`OnCollisionEnter2D`, `OnCollisionStay2D`, + `OnCollisionExit2D` and the three `OnTrigger` equivalents). +* `OnMouseDown` and `OnMouseUp` on an object with a 2D collider, with the + pointer placed in the world by the main camera. `OnApplicationPause(bool)` + and `OnApplicationFocus(bool)` when the game is paused and resumed. +* Coroutines with `yield return null`, `WaitForSeconds`, `WaitForEndOfFrame` + and `WaitForFixedUpdate`, a `Start` written as a coroutine, + `StopCoroutine` and `StopAllCoroutines`. +* `Invoke`, `InvokeRepeating`, `CancelInvoke` and `IsInvoking`. +* `GameObject` and `Transform` hierarchies, `Instantiate` with or without a + position, a rotation and a parent, `Destroy` with an optional delay, `SetActive`, tags and layers, + `Find`, `FindWithTag`, `FindGameObjectsWithTag`, `FindObjectOfType`, + `AddComponent`, and `GetComponent` with its `InChildren` and `InParent` + forms. +* Serialized fields of scripts, as the Inspector stored them: numbers, + strings, enums, vectors, colors, arrays and lists, structs, and references to objects, components, prefabs, sprites, audio + clips and text assets. +* Prefabs, prefab instances placed in a scene with their property overrides, + and nested prefabs. +* `SceneManager.LoadScene` by name or by build index, `Time` with + `timeScale` and `timeSinceLevelLoad`, `Random`, `Screen.width` and + `Screen.height`, `Debug.Log` and its warning and error forms, and + `UnityEngine.Assertions.Assert`. +* `Application.platform` and `Application.isMobilePlatform`, answered from the + device the game runs on: `IPhonePlayer` on iOS, `Android` on Android, + `WebGLPlayer` in a browser, and the desktop's own player on macOS, Windows + and Linux. The simulator reports the desktop it runs on. + +==== Rendering + +* `SpriteRenderer` with sorting layers and orders, flipping and tinting. +* Sprites imported as single images, sprite sheets sliced in the Sprite + Editor, pivots and pixels per unit. A polygon sprite is cut to its outline + when the project is built and drawn as that image. +* An orthographic `Camera` with its background color and the conversions + between screen, viewport and world points. + +==== Animation + +* `Animator` with the controllers and clips of the project. The build + compiles each `.controller` and `.anim` file into code, so nothing is + parsed on the device. +* State machines with float, integer, bool and trigger parameters, + transitions with conditions and exit time, and transitions from Any State. +* Curves that animate a sprite renderer's sprite, color and enabled flag, a + transform's position, scale and rotation about z, and whether a GameObject + is active. +* Animation events, which call the method they name on the scripts of the + animated object. +* From a script: `Play`, `CrossFade`, `SetFloat`, `SetInteger`, `SetBool`, + `SetTrigger`, `ResetTrigger` and their getters, `speed`, + `GetCurrentAnimatorStateInfo`, `IsInTransition`, `StringToHash`, and + assigning `runtimeAnimatorController` to swap the controller. + +==== Tile maps + +* `Grid`, `Tilemap` and `TilemapRenderer`, with the cells the Tile Palette + painted. +* `TilemapCollider2D`, alone or merged into outlines by a + `CompositeCollider2D`. +* From a script: `GetTile`, `SetTile`, `HasTile`, `GetSprite`, + `ClearAllTiles`, `WorldToCell`, `CellToWorld` and `GetCellCenterWorld`. + +==== Particles + +* `ParticleSystem`, simulated on the CPU from a seeded sequence, so an effect + plays the same way on every target. +* The main module, emission by rate and in bursts, the emitter shape, and the + velocity, color, size and rotation over lifetime modules, as the scene + stores them. +* From a script: `Play`, `Stop`, `Pause`, `Clear` and `Emit`, the playing + state and `particleCount`, and the `main`, `emission` and + `velocityOverLifetime` modules. + +==== Physics + +* `Rigidbody2D`: dynamic, kinematic and static bodies, velocity, forces and + torques, `MovePosition` and `MoveRotation`, mass, drag, gravity scale, + `constraints` and `collisionDetectionMode`. +* Box, circle, polygon, capsule and edge colliders, tile map and composite + colliders, triggers, and `Collision2D` with its contacts. +* `PlatformEffector2D` for one-way platforms. +* Physics materials (friction and bounciness), the project's gravity and the + layer collision matrix, with `Physics2D.IgnoreLayerCollision` to change it + from a script. +* Casts: `Physics2D.Raycast`, `Linecast`, `CircleCast`, `BoxCast` and + `CapsuleCast`. Each has the form that returns the nearest hit, the `All` + form, the `NonAlloc` form and the form that takes a `ContactFilter2D` with + an array or a list for the results. +* Overlap tests: `OverlapPoint`, `OverlapCircle`, `OverlapBox`, `OverlapArea` + and `OverlapCapsule`, in the same four forms. +* `RaycastHit2D`, `ContactFilter2D` and `LayerMask`, including a `LayerMask` + field set in the Inspector, with `Physics2D.queriesHitTriggers`, + `queriesStartInColliders` and `SyncTransforms`. +* On a collider or a body: `OverlapPoint`, `IsTouching`, `IsTouchingLayers` + and `Cast`, and `Collider2D.bounds`. + +A cast isn't stepped along its path. The runtime solves each one in closed +form, so the distance, the normal and the point it reports are exact to the +rounding of the arithmetic, and a fast or thin shape can't slip past a +collider. A query sees bodies where the last physics step put them; call +`Physics2D.SyncTransforms` after a script moves a transform and before it +asks. + +==== Input + +* `GetKey`, `GetKeyDown` and `GetKeyUp` by `KeyCode` or by name, `anyKey` and + `anyKeyDown`. +* The axes and buttons the project's input settings define on keys, through + `GetAxis`, `GetAxisRaw` and `GetButton`. +* The mouse: `GetMouseButton`, `GetMouseButtonDown`, `GetMouseButtonUp` and + `mousePosition`, and the `Mouse X` and `Mouse Y` axes for its movement. +* Touch: `touchCount`, `touches` and `GetTouch`, with a `Touch` that carries + its `fingerId`, `position`, `deltaPosition`, `tapCount` and a `phase` of + `Began`, `Moved`, `Stationary` or `Ended`. `multiTouchEnabled` and + `simulateMouseWithTouches` work as documented. + +==== User interface, audio and data + +* A `Canvas` in Screen Space, overlay or camera, with a `CanvasScaler` and + `RectTransform` anchoring. +* `Text` with size, style, alignment, wrapping and best fit. +* TextMesh Pro text, as `TMP_Text`, `TextMeshPro` and `TextMeshProUGUI`: + `text`, `SetText`, size, style, alignment, wrapping and auto sizing, drawn + in the platform's font. +* `Image`, simple or filled horizontally or vertically (`fillAmount` drives a + health bar). +* `Button` with its color tint states, the `onClick` calls set up in the + Inspector, and `onClick.AddListener` from a script. +* `AudioSource` and `AudioClip`, played through Codename One's media API: + `Play`, `PlayOneShot`, `Stop`, `Pause`, `loop`, `volume` and + `PlayClipAtPoint`. +* `PlayerPrefs` for integers, floats and strings, saved on the device. +* `Resources.Load` for text assets, images imported as one sprite, audio + clips and prefabs, and `TextAsset` with `text` and `bytes`. + +==== Follow cameras + +* Cinemachine's `CinemachineBrain` and `CinemachineVirtualCamera` with a + `CinemachineFramingTransposer` or a `CinemachineTransposer` body: a camera + that follows its target with the offset, damping, dead zone and soft zone + the scene sets. +* Several virtual cameras by priority, with a cut from one to the next, and + `Follow` assigned from a script. + +[[unity-interop-determinism]] +=== The same game on every target + +Arithmetic on `float` gives the same result on every target, and +`UnityEngine.Random` produces the same sequence from the same seed. A game +that's given a fixed seed and the same input plays the same way in the +simulator, in a native build and in the browser. + +That takes work in two places, because the targets don't agree by default. +The C compiler that builds the iOS and desktop binaries may fuse a multiplication +and an addition into one instruction that rounds once instead of twice, so the +native builds turn that contraction off. A JavaScript number is a double, so +on the JavaScript port every `float` result is rounded back to single +precision before it's used. + +The evidence is a trace comparison. Each game in the next section was run +from a fixed seed with scripted input on three targets: a JVM, the C code +ParparVM generates, and the JavaScript port under Node.js. Each run wrote a +trace of the game's log lines, its object counts and its draw list, with +positions in hundredths of a pixel and rotations in hundredths of a degree. +The three traces of each game are byte for byte the same: 3,503 lines for +Fruitopia, 220 for Asteroids, 852 for Pong, 244 for Snake and 445 for +Sokoban. Fruitopia's run goes through its menu and its lobby, plays a level +to the flag and returns to the lobby, 1,400 frames of animation, particles, +tile map collisions and a follow camera. Asteroids moves +more than 1,100 objects every frame, so one differently rounded multiply +shows up in its trace within a few hundred frames. + +`Random` is seeded from the clock unless you set a seed. To make a run +repeatable, set the `unity.seed` display property before the game starts, for +example `Display.getInstance().setProperty("unity.seed", "7")` at the top of +the main class's `init` method. + +[[unity-interop-games]] +=== Games that run + +Five open source Unity games build and run with no change to their scripts, +scenes or assets. Each is a repository its author published under the MIT +license. They aren't part of Codename One; clone one and import it to try it. + +Every image in this section is a frame the Codename One runtime drew from the +game's own scenes and sprites, at 960 by 540 pixels. + +==== Fruitopia + +https://github.com/thisshrek/Fruitopia[`thisshrek/Fruitopia`], MIT license, +copyright 2024 Mohamed Kharrat. Twelve scripts, 733 lines. Its art is the +Pixel Adventure set by Pixel Frog, released under CC0 and credited in the +game's README. + +A pixel-art platformer with a menu, a lobby and levels to run through: +collect the fruit, avoid the traps, reach the flag. It's the widest test of +the five. Its levels are tile maps with composite colliders and one-way +platforms. Its hero, fruit, traps and menus are driven by 65 `Animator` +components that play 40 clips. The dust behind the hero is a particle +system, the labels are TextMesh Pro text, and a Cinemachine virtual camera +follows the hero. The hero moves through `Rigidbody2D` and finds the ground +with `Physics2D.OverlapBox`. + +image::img/unity-interop-fruitopia-level.png["Fruitopia running on the Codename One runtime: the frog hero in a tile map level with pineapples to collect, a spiked ball and the fruit counter",scaledwidth=70%] + +The text in these frames is in the platform's font rather than the game's +font asset, which is how the runtime draws TextMesh Pro text. + +image::img/unity-interop-fruitopia-menu.png["The Fruitopia menu running on the Codename One runtime: the title and three buttons over a tile map backdrop",scaledwidth=70%] + +The build reports six warnings for this project, and each names something +the project itself is missing or does that Unity also drops. Two levels place +a prefab whose file isn't in the repository, one object carries a script +that isn't under `Assets`, and three objects carry a static class as a +component. The menu, the lobby and the first level play from start to finish. + +==== Asteroids + +https://github.com/sdxsharp/unity-asteroids[`sdxsharp/unity-asteroids`], MIT +license, copyright 2022 `sdxsharp`. Five scripts, 546 lines, saved by Unity +2021.3. + +An arcade shooter that keeps between 1,157 and 1,180 objects alive, most of +them the stars of its background. It exercises rigid bodies driven by +velocity, polygon and circle colliders, collision messages, prefabs +instantiated and destroyed as the game runs, seeded `Random`, the input axes +and a canvas with a score line. + +image::img/unity-interop-asteroids.png["Asteroids running on the Codename One runtime: the ship, three asteroids, the star field and the score line",scaledwidth=70%] + +==== Pong + +https://github.com/MalachiMackie/Unity-Pong[MalachiMackie/Unity-Pong], MIT +license, copyright 2021 Malachi Mackie. Seven scripts, 557 lines, saved by +Unity 2021.1. + +A two-player ball game. It uses prefab instances with property overrides, +physics materials for the bounce, audio clips, C# events and delegates with +tuple payloads, LINQ and switch expressions. + +image::img/unity-interop-pong.png["Pong running on the Codename One runtime: two paddles, the ball and the three score counters",scaledwidth=70%] + +==== Snake + +https://github.com/Wesley-Oliveira/Classic-Snake-Game-Unity-2D[Wesley-Oliveira/Classic-Snake-Game-Unity-2D], +MIT license, copyright 2020 Wesley Oliveira. Two scripts, 184 lines, saved +by Unity 2019.3. + +A grid game with a menu. Its menu is canvas buttons whose `onClick` calls +were set up in the Inspector, its record is kept in `PlayerPrefs`, and the +snake moves on a coroutine that respects `Time.timeScale`. + +image::img/unity-interop-snake.png["Snake running on the Codename One runtime: the snake, the food, the score and the saved record",scaledwidth=70%] + +==== Sokoban + +https://github.com/juwalbose/UnityTileBasedSokoban[`juwalbose/UnityTileBasedSokoban`], +MIT license, copyright 2017 Juwal Bose. One script, 264 lines, saved by +Unity 2017.1. + +A tile puzzle. The level is a text file loaded with `Resources.Load`, split +and parsed into an `int[,]` grid, with the tile objects tracked in a +`Dictionary`. The script creates every tile as a +`GameObject` with a `SpriteRenderer` it adds itself. The hero and the balls +are polygon sprites, and the Restart button reloads the scene through a persistent call to +`SceneManager.LoadScene`. + +image::img/unity-interop-sokoban.png["Sokoban running on the Codename One runtime, part way through the level: the triangular hero, three balls and three target tiles",scaledwidth=70%] + +image::img/unity-interop-sokoban-complete.png["Sokoban running on the Codename One runtime with the level complete: each of the three balls is on a target tile",scaledwidth=70%] + +[[unity-interop-measurements]] +=== Measurements + +The table covers the four smaller games above; Fruitopia follows it. The +numbers measure the game and the +compatibility runtime alone, in a harness that runs the game without a +display: the same scripted run that produced the traces, three runs each, +median reported. A shipped application adds the Codename One port for its +platform on top. + +[cols="2,1,1,1,1",options="header"] +|=== +|Game |Objects alive |Native executable |JavaScript, gzip |Peak memory, native +|Asteroids |1,157 to 1,180 |1.33 MB |246 KB |15.7 MB +|Pong |13 |1.36 MB |244 KB |6.2 MB +|Snake |26 to 31 |1.33 MB |243 KB |5.1 MB +|Sokoban |40 |1.30 MB |241 KB |4.8 MB +|=== + +* *Native executable* is the stripped macOS binary of the game, the runtime + and the Java class library, translated to C by ParparVM and linked with + link-time optimization. +* *JavaScript, gzip* is the script the JavaScript port loads for the same + code, compressed with `gzip -9`. +* *Peak memory* is the most physical memory that native executable held + over an 1,800-frame run, as `/usr/bin/time -l` reports it. + +The translated code is small next to the runtime it runs on. Sokoban's +classes and the whole compatibility runtime pack into a 268 KB jar, of which +the runtime is 252 KB. The games were measured against successive builds of +the runtime, so a difference of 2% or 3% between rows isn't +significant. + +Fruitopia is the largest of the five. These figures are from its scripted +1,400-frame run in the same harness. Between 59 and 80 objects are alive and each frame issues 350 +to 500 draw commands. + +* The compatibility runtime it ships with is 382 KB. The rest of its jar is + the game's own audio. +* The native executable, linked with link-time optimization and stripped, is + 2.67 MB. +* The JavaScript for the same code is 387 KB after `gzip -9`. +* Peak memory over the run was about 10 MB for the native executable, about + 99 MB on a JVM and about 176 MB under Node.js. +* On the JVM the simulation allocated under 2 KB per frame over the whole + run, three scene loads included. + +On speed: Asteroids, the heaviest of the four in the table, spent under half a millisecond +of CPU per frame in the native build, simulation and draw list together, on a +development machine that was busy with other builds. The other three were +below what the timer resolves. Treat that as an order of magnitude and +measure your own game on the device you ship for. + +[[unity-interop-scope]] +=== Scope and porting notes + +The compatibility layer targets 2D games built from sprites, tile maps, 2D +physics, animation controllers, particles, a canvas interface and scripts. +This section lists what's outside that surface +today, what the build does when it meets each item, and what to do about it. + +==== How the build tells you + +A port doesn't fail at run time for a reason the build could have named. +There are four places it speaks up: + +* A `UnityEngine` member the runtime doesn't have is a C# compiler error with + the file and the line. +* A .NET base library member the runtime doesn't have stops the build at the + translation step, with every such member listed at once and, under each, + the script file, the line and the method that uses it. +* A component a scene has and the runtime doesn't is a warning that names the + scene, the GameObject and the component. The component is left out and the + rest of the scene still builds. +* A script method Unity would call by name and the runtime doesn't is a + warning with the class, the method, the file and the line. + +==== Rendering notes + +* *3D.* Meshes, lights, 3D physics, shaders and materials other than the + default sprite material are outside the scope. Their components are left + out with a warning. A perspective camera is made orthographic, and a + rotation about the x-axis or the y-axis is dropped, each with a warning. +* *Animation controllers.* The first layer of a controller is played, and + the build warns when a controller has more. A state that plays a blend tree + shows nothing, and a transition into a sub-state machine is left out, each + with a warning. Give such a state a plain clip, or drive the blend from a + script with `Play` and `CrossFade`. An override controller plays nothing, + with a warning; assign `runtimeAnimatorController` from a script instead. +* *Animated properties.* A clip animates a sprite renderer's sprite, color + and enabled flag, a transform's position, scale and rotation about z, and + a GameObject's active flag. A curve on another property is left out with a + warning that names it; set that property from an animation event or from + `Update`. +* *Timelines* aren't run. Sequence the same steps in a coroutine. +* *Particle modules.* The main, emission, shape, and velocity, color, size + and rotation over lifetime modules are simulated. Another module that's + switched on, such as Noise, Collision, Trails, Sub Emitters or Texture + Sheet Animation, does nothing, and the build warns for each. A system that + emits from a mesh, a donut or a sprite emits from its own position, one + that stretches its particles or draws meshes draws plain sprites facing the + camera, and a system with `Prewarm` on starts empty. Of the modules a script can + reach, `main`, `emission` and `velocityOverLifetime` are in the API. +* *Scripted tiles.* A rule tile or an animated tile isn't run. The cells a + scene was painted with show the sprites the editor chose for them, and a + cell a script sets with such a tile shows its default sprite, with a note. + Use plain tiles for cells a script changes. +* *Line and trail renderers, sprite masks.* The components are left out with + a warning. An effect made of short-lived sprite objects that a script + instantiates and destroys carries over as it is. +* *Sliced and tiled sprites.* A nine-sliced or tiled `SpriteRenderer` or + `Image` draws its sprite stretched, with a warning. A radial fill draws + whole. +* *Built-in and package sprites.* The sprites that belong to the Unity editor + or to a Unity package aren't in the project, so the build draws a plain + rectangle or a white disc in their place and says so. Add your own sprite + to the project to control the look. + +==== Physics notes + +* *Queries.* The casts and overlap tests are listed under + <>. `Physics2D.Distance`, + `Collider2D.Raycast` and `Physics2D.GetRayIntersection` aren't in the API, + so a script that calls one is a compile error. `Physics2D.Raycast` with a + layer mask or a `ContactFilter2D` answers what `Collider2D.Raycast` is + usually asked, and `OverlapPoint` at the pointer's world position does the + job of `GetRayIntersection` in a 2D scene. +* *Edge radius.* An `EdgeCollider2D` with an edge radius collides as a thin + line, with a warning. Use a capsule or a box collider where the thickness + matters. +* *Continuous collision detection.* A body set to Continuous is swept so that + it can't pass through a thin collider. The physics engine decides this for + the whole world: one Continuous body makes it sweep the others too. +* *Joints and other effectors.* Joints are left out with a warning. + `PlatformEffector2D` is the effector that's supported; a collider used by + another effector collides as itself, with a warning. Apply the force of an + area or point effector from `OnTriggerStay2D`. +* *Automatic mass.* A body set to *Use Auto Mass* uses the mass the file + records, with a warning. + +==== Script notes + +* *Messages.* The runtime calls the messages listed under + <>. A method named for + another Unity message, such as `OnBecameInvisible`, `OnMouseDrag` or + `OnGUI`, compiles and is never called. The build warns for each one, with + the class, the method, the file and the line, so the list is in the build + output. A drag can be read in `Update` from `Input.GetMouseButton` and + `Camera.ScreenToWorldPoint`. +* *Coroutine waits.* `WaitUntil`, `WaitWhile` and `WaitForSecondsRealtime` + aren't in the API. A `while` loop around `yield return null` does the same + job. +* *Scenes.* One scene is loaded at a time. `DontDestroyOnLoad` keeps a root + object across a load, with everything below it and its running coroutines + and pending `Invoke` calls. Called for a child object it logs a message and + keeps nothing, as in Unity. +* *The base library.* The runtime implements the part of .NET listed above, + not the whole library. `List.Sort`, `StringBuilder`, format strings such as + `ToString("F2")`, `float.Parse`, `HashSet`, `Queue`, `Stack`, + `DateTime` and `System.Random` are the ones a port meets first. The + translation step lists each use; the usual replacement is a few lines in + the script, such as a hand-written insertion sort or `UnityEngine.Random`. +* *Static fields of generic types.* Each constructed type has its own, as + in .NET: `Singleton.Instance` and + `Singleton.Instance` are two fields, and a static constructor + runs once for each. Three shapes can't be translated, and the build stops + with the type, the method and the line: an instance method of a generic + struct that reads its static fields, a virtual or interface generic method + that reaches them through its own type argument, such as `Slot.Value` + inside `virtual T Read()`, and a type that names itself with an + ever-larger argument, such as `Chain` calling `Chain>`. Make + the struct a class, make the method non-virtual, or keep that state in a + non-generic class. +* *Reflection, `System.IO`, threads, `async` and `await`, networking and + `unsafe` code.* None of these are translated. Keep saved data in + `PlayerPrefs`, load bundled data with `Resources.Load`, and run timed work + in coroutines. Networking belongs on the Java side of the application. +* *Compiled plugins.* Only C# source is translated. The import lists every + `.dll` and native library under `Assets` as not built. Use the source of + the library where its license allows. +* *Packages from the Package Manager.* The build compiles what's under + `Assets`. Copy the source of a package the scripts need into `Assets`. +* *Serialized fields.* A field whose type is a serializable class of your + own keeps the value its constructor gave it, and the build warns with the + field's name. Make the type a struct, which is read, or assign the value in + `Awake`. +* *Editor scripts and custom inspectors.* Scripts in an `Editor` directory + are left out, the way Unity leaves them out of a player build. + +==== Input notes + +* *The Input System package.* The runtime implements the `UnityEngine.Input` + class. A script written against the newer Input System package doesn't + compile; read the same keys, axes and touches through `Input`. +* *Touch phases.* A touch goes through `Began`, `Moved`, `Stationary` and + `Ended`. `Canceled` is never reported, so move the cleanup a + script keeps there into `Ended`. +* *Joystick axes and the scroll wheel.* An axis the input settings bind to a + joystick reads as zero, with a note, and so does `Mouse ScrollWheel`. The + keys bound to the same axis work. +* *Space and Return.* The desktop port reports the space bar and the Enter + key as one fire button, so on the desktop both `KeyCode.Space` and + `KeyCode.Return` read as pressed when either is. A game that gives the two + keys different meanings on the same screen sees both. + +==== User interface notes + +* *Controls.* `Canvas`, `CanvasScaler`, `Text`, `Image`, `Button`, + `GraphicRaycaster` and `EventSystem` are supported. Another control of + Unity's interface package -- a slider, a toggle, an input field, a scroll + view or a layout group -- is left out with a warning + that names the scene, the GameObject and the Unity type, and the build + continues. A script field that pointed at the control is `null`, so guard + the script's use of it. Replace the control with buttons and images, or + show that screen with Codename One components around the game, where every + Codename One control is available. +* *World Space canvases* aren't drawn, with a warning. +* *Fonts.* Text is drawn in the platform's font at the size, style and + alignment the scene sets. A font asset the project names isn't used, which + the build notes for each text. +* *TextMesh Pro.* TextMesh Pro text is drawn the same way: the platform's + font at the size asked for, without the font asset, its material, an + outline or rich text tags, with a note for the scene. Tags in a string are + therefore drawn as written; keep them out of text shown here. +* *Button transitions.* Color tint is supported and changes at once rather + than fading. A sprite swap or animation transition leaves the button + looking the same in every state; it still works. +* *Keyboard navigation between controls* isn't implemented. Controls respond + to the pointer. + +==== Camera notes + +* *Cinemachine.* A virtual camera follows its target through a framing + transposer or a transposer, and the brain cuts between virtual cameras. + Blends, aim behaviors, noise and extensions such as a confiner aren't + implemented, which the build notes once for a scene. A look-ahead time on a + framing transposer is ignored, with a note. To keep the camera inside a + level, clamp its position in `LateUpdate`. + +==== Audio and asset notes + +* *Formats.* Clips are played by each platform's own media player, so try + the game's sound files on every target you build for. +* *Pitch and looping.* A clip plays at its recorded speed whatever + `AudioSource.pitch` says, and a looped clip restarts when it ends, which can + leave a short gap. +* *`Resources`.* `Resources.Load` finds text assets, single-sprite images, + audio clips and prefabs. `LoadAll` isn't in the API. An image sliced into + several sprites isn't found by `Resources.Load`, with a note; refer to its + sprites from a serialized field instead. +* *Asset bundles and the `Addressables` package* aren't supported. Refer to assets from a + scene, a prefab or a `Resources` folder. +* *`Texture2D`* and run-time texture generation aren't in the API. +* *File names.* Images and sounds become application resources, which have + no folders. Two images with the same file name in different folders both + ship, each under a name of its own, and every sprite shows its own image. + Two audio clips with the same file name still collide, and the build warns + about each pair. Rename one. +* *Scripts that can't be components.* A scene that attaches a static class, + an abstract class or a class that isn't a `MonoBehaviour` as a component + gets a warning, and the component is left out, as Unity leaves it out. + +[[unity-interop-troubleshooting]] +=== Troubleshooting + +*The build can't find the .NET SDK.* The message names the three places it +looked: + +---- +src/main/unity holds a Unity project, and compiling its C# scripts needs the .NET SDK, which was not found. +---- + +Install the SDK and put `dotnet` on the `PATH`, or pass +`-Dcn1.unity.dotnet=/path/to/dotnet`. A .NET runtime without the SDK isn't +enough, and the build says so when `dotnet --version` reports no SDK. An SDK +older than version 6 is reported as too old. + +*The C# scripts don't compile.* A script uses a `UnityEngine` type or +member outside the runtime's API. The build prints the C# compiler's errors, +each with its file, line and column: + +---- +The C# scripts in src/main/unity/Assets did not compile: +/work/mygame/common/src/main/unity/Assets/Scripts/Player.cs(14,38): error CS0117: 'Physics2D' does not contain a definition for 'Distance' +---- + +The complete compiler output is in `dotnet.log`. Change the script, or check +<> for the replacement. + +*The translation step fails.* The scripts compiled, and they use +a part of the .NET base library the runtime doesn't have. The translator +lists every missing member, and under it each place that uses it: + +---- +The C# scripts in src/main/unity could not be translated: +error: the runtime library lacks 2 members the code uses: + class com/codename1/unitycompat/system/text/StringBuilder + used by HighScores.cs:17, in HighScores::Print() : string + method com/codename1/unitycompat/system/collections/generic/List_1.Sort()V + used by HighScores.cs:12, in HighScores::Start() : void +---- + +The class after `unitycompat/system/` is the .NET class, and each `used by` +line is the script file, the line and the method to change. + +*A control or a package component is left out.* A scene or prefab has a +component whose script isn't under `Assets`: an interface control outside the +supported set, or a component of a Package Manager package. The build goes on +and warns, once for the component and once for each script field that +referred to it: + +---- +First.unity: GameObject 'Gauge' (component file ID 202) has a UnityEngine.UI.Slider component (script GUID 67db9e8f0e2ae9c40bc1e2b64352a6b4), and the compatibility runtime has no UnityEngine.UI.Slider; the component was left out, and whatever refers to it is left null +First.unity: field gauge of the Keeper script on GameObject 'Keeper' (component file ID 102) refers to a UnityEngine.UI.Slider component (script GUID 67db9e8f0e2ae9c40bc1e2b64352a6b4) of GameObject 'Gauge' (component file ID 202), which was left out; it was left null +---- + +Replace the control, or make the script check the field for `null`. For a +package whose source you have, copying its scripts into `Assets` compiles +them with the project. A scene the build settings list that isn't in the +project is left out with a warning too. + +*A script method is never called.* Unity calls some methods by name, and the +runtime sends the ones listed under +<>. The build warns about +any other: + +---- +Keeper.OnBecameInvisible (Keeper.cs:98): Unity calls this method by its name and the compatibility runtime does not send the OnBecameInvisible message; the method is never called +---- + +*A component is missing in the running game.* Read the warnings of the last +build that compiled the Unity project; a build that found the project +unchanged doesn't repeat them. `translate.log` and `scene-compiler.log` in +`common/target/unity` keep them, each prefixed `warning:` or `note:`. + +*The game looks or behaves differently.* Check, in this order: the warnings +in those two logs; the notes there about substituted sprites and fonts; and +whether a script relies on something listed under +<>. `Debug.Log` output appears in the simulator's +console. + +*The main class wasn't updated by a second import.* The import leaves a main +class you changed after the first import untouched, and says so. Delete the +class and import again to regenerate it. + +=== Independent implementation + +The runtime is an independent implementation of the parts of the Unity +scripting API this chapter lists, written from Unity's public documentation +and from the observable behavior of projects. It contains no Unity code and +no Unity assemblies, and the build doesn't read or need a Unity installation. +The same holds for the two packages the runtime answers for. Its TextMesh Pro +and Cinemachine classes are Codename One's own implementations of the parts +of those documented APIs this chapter lists, and contain no code from either +package. +Codename One isn't affiliated with or endorsed by Unity Technologies. Unity +is a trademark of Unity Technologies. + +Your scripts and assets stay under the terms you hold them under. Assets from +the Unity Asset Store and other third parties have their own license terms. +Check that those terms allow use outside the Unity engine before you ship a +port that includes them. diff --git a/docs/developer-guide/developer-guide.asciidoc b/docs/developer-guide/developer-guide.asciidoc index fd86060c6aa..db4ecf29826 100644 --- a/docs/developer-guide/developer-guide.asciidoc +++ b/docs/developer-guide/developer-guide.asciidoc @@ -251,6 +251,8 @@ include::Working-With-Javascript.asciidoc[] include::Android-Interop.asciidoc[] +include::Unity-Interop.asciidoc[] + include::Working-with-Mac-OS-X.asciidoc[] include::Working-with-Mac-Catalyst.asciidoc[] diff --git a/docs/developer-guide/img/unity-interop-asteroids.png b/docs/developer-guide/img/unity-interop-asteroids.png new file mode 100644 index 00000000000..8717db67c2e Binary files /dev/null and b/docs/developer-guide/img/unity-interop-asteroids.png differ diff --git a/docs/developer-guide/img/unity-interop-fruitopia-level.png b/docs/developer-guide/img/unity-interop-fruitopia-level.png new file mode 100644 index 00000000000..c0ac92a419b Binary files /dev/null and b/docs/developer-guide/img/unity-interop-fruitopia-level.png differ diff --git a/docs/developer-guide/img/unity-interop-fruitopia-menu.png b/docs/developer-guide/img/unity-interop-fruitopia-menu.png new file mode 100644 index 00000000000..520971a6857 Binary files /dev/null and b/docs/developer-guide/img/unity-interop-fruitopia-menu.png differ diff --git a/docs/developer-guide/img/unity-interop-fruitopia.png b/docs/developer-guide/img/unity-interop-fruitopia.png new file mode 100644 index 00000000000..2cc3f5f9fb5 Binary files /dev/null and b/docs/developer-guide/img/unity-interop-fruitopia.png differ diff --git a/docs/developer-guide/img/unity-interop-pong.png b/docs/developer-guide/img/unity-interop-pong.png new file mode 100644 index 00000000000..e9e5e20d030 Binary files /dev/null and b/docs/developer-guide/img/unity-interop-pong.png differ diff --git a/docs/developer-guide/img/unity-interop-snake.png b/docs/developer-guide/img/unity-interop-snake.png new file mode 100644 index 00000000000..a4fa4cfe16c Binary files /dev/null and b/docs/developer-guide/img/unity-interop-snake.png differ diff --git a/docs/developer-guide/img/unity-interop-sokoban-complete.png b/docs/developer-guide/img/unity-interop-sokoban-complete.png new file mode 100644 index 00000000000..cb799cbf707 Binary files /dev/null and b/docs/developer-guide/img/unity-interop-sokoban-complete.png differ diff --git a/docs/developer-guide/img/unity-interop-sokoban.png b/docs/developer-guide/img/unity-interop-sokoban.png new file mode 100644 index 00000000000..2ab7015c2f2 Binary files /dev/null and b/docs/developer-guide/img/unity-interop-sokoban.png differ diff --git a/docs/developer-guide/languagetool-accept.txt b/docs/developer-guide/languagetool-accept.txt index c71b484ddbf..b9d80eddf61 100644 --- a/docs/developer-guide/languagetool-accept.txt +++ b/docs/developer-guide/languagetool-accept.txt @@ -803,3 +803,8 @@ Toolchain # (performance.asciidoc). inlining inlined +Sokoban +Juwal +Fruitopia +Kharrat +Cinemachine diff --git a/docs/website/data/port_status.json b/docs/website/data/port_status.json index 72e56c8f368..75d68e0a811 100644 --- a/docs/website/data/port_status.json +++ b/docs/website/data/port_status.json @@ -129,6 +129,22 @@ "AndroidCompatKotlinScreenshotTest" ] }, + { + "id": "unity-compatibility", + "category": "Application runtime", + "name": "Unity 2D projects (UnityEngine and C# scripts)", + "description": "Runs an unmodified Unity 2D project -- C# scripts, scenes and prefabs, sprites, 2D physics, the UI canvas, tilemaps, animators, particle systems, TextMeshPro and Cinemachine -- through the Unity compatibility runtime, drawn by the GPU.", + "tests": [ + "UnityCompatSpritesScreenshotTest", + "UnityCompatPhysicsScreenshotTest", + "UnityCompatCanvasScreenshotTest", + "UnityCompatTilemapScreenshotTest", + "UnityCompatAnimatorScreenshotTest", + "UnityCompatParticlesScreenshotTest", + "UnityCompatTextMeshProScreenshotTest", + "UnityCompatCinemachineScreenshotTest" + ] + }, { "id": "ads", "category": "Application runtime", @@ -1584,6 +1600,38 @@ { "pattern": "AndroidCompatKotlin", "test": "AndroidCompatKotlinScreenshotTest" + }, + { + "pattern": "UnityCompatSprites", + "test": "UnityCompatSpritesScreenshotTest" + }, + { + "pattern": "UnityCompatPhysics", + "test": "UnityCompatPhysicsScreenshotTest" + }, + { + "pattern": "UnityCompatCanvas", + "test": "UnityCompatCanvasScreenshotTest" + }, + { + "pattern": "UnityCompatTilemap", + "test": "UnityCompatTilemapScreenshotTest" + }, + { + "pattern": "UnityCompatAnimator", + "test": "UnityCompatAnimatorScreenshotTest" + }, + { + "pattern": "UnityCompatParticles", + "test": "UnityCompatParticlesScreenshotTest" + }, + { + "pattern": "UnityCompatTextMeshPro", + "test": "UnityCompatTextMeshProScreenshotTest" + }, + { + "pattern": "UnityCompatCinemachine", + "test": "UnityCompatCinemachineScreenshotTest" } ] } diff --git a/maven/README.adoc b/maven/README.adoc index 62aabbecbf7..769a6d6ac1a 100644 --- a/maven/README.adoc +++ b/maven/README.adoc @@ -32,6 +32,14 @@ publishing scripts enforce. NOTE: SNAPSHOT versions do not trigger a release. +The release needs a .NET SDK, and network access to nuget.org. `codenameone-cil-translator` +and `codenameone-unity-compat` are published with everything else, in the same reactor pass +and at the same version, and the second compiles C# and restores the `NETStandard.Library.Ref` +reference pack to do it. The workflow installs the SDK, passes `-Dunity-compat` to turn on the +profile both modules live in, and stops before building anything if `dotnet` is absent. +Bumping the version needs neither: `update-version.sh` reaches the two modules without the +profile, because `versions:set` reads the modules of every profile. + ==== Step 1: Update Version to New Release Version [source,bash] diff --git a/maven/build-engine/spotbugs-exclude.xml b/maven/build-engine/spotbugs-exclude.xml index 25d77303053..884c8dfff18 100644 --- a/maven/build-engine/spotbugs-exclude.xml +++ b/maven/build-engine/spotbugs-exclude.xml @@ -291,4 +291,12 @@ + + + + + diff --git a/maven/build-engine/src/main/java/com/codename1/maven/BackendPackager.java b/maven/build-engine/src/main/java/com/codename1/maven/BackendPackager.java index 66c42efdc5a..30acc29c65b 100644 --- a/maven/build-engine/src/main/java/com/codename1/maven/BackendPackager.java +++ b/maven/build-engine/src/main/java/com/codename1/maven/BackendPackager.java @@ -963,6 +963,10 @@ private void link(File translated, File binary) // provably miscompiles the output without these. "-fwrapv", "-fno-strict-aliasing", "-fno-builtin-fmod", "-fno-builtin-fmodf", + // Java evaluates a * b + c as two rounded operations. Clang fuses + // them into one multiply-add where the CPU has one (arm64), which + // rounds once and answers differently from every other target. + "-ffp-contract=off", // One section per function and per object, so that the link // below can leave out what nothing reaches. The natives are // compiled whole; without this a server that signs nothing diff --git a/maven/build-engine/src/main/java/com/codename1/maven/GradleConversion.java b/maven/build-engine/src/main/java/com/codename1/maven/GradleConversion.java index 3fc39c854db..70338d573c7 100644 --- a/maven/build-engine/src/main/java/com/codename1/maven/GradleConversion.java +++ b/maven/build-engine/src/main/java/com/codename1/maven/GradleConversion.java @@ -1073,7 +1073,11 @@ private static void copyPlatformResources(ProjectLayout from, ProjectLayout to) static final java.util.Set PLUGIN_SUPPLIED = new java.util.HashSet(java.util.Arrays.asList( "codenameone-core", "codenameone-javase", "java-runtime", "codenameone-backend", "cn1-binaries-javase", "codenameone-css-cli", "codenameone-android", "codenameone-ios", "codenameone-javascript", - "codenameone-maven-plugin")); + "codenameone-maven-plugin", + // Added by the plugin for a project with src/main/unity, at its own + // version, together with the translator of that version: a + // declaration carried over from the pom could name another one. + "codenameone-unity-compat")); /// `build.gradle.kts` lines for the dependencies a module's pom declares /// itself, leaving out the framework, which the plugin adds. A dependency of diff --git a/maven/build-engine/src/main/java/com/codename1/maven/UnityProjectBuilder.java b/maven/build-engine/src/main/java/com/codename1/maven/UnityProjectBuilder.java new file mode 100644 index 00000000000..a255d158ff5 --- /dev/null +++ b/maven/build-engine/src/main/java/com/codename1/maven/UnityProjectBuilder.java @@ -0,0 +1,1482 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.maven; + +import com.codename1.build.Log; +import com.codename1.builders.BuildException; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Enumeration; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.zip.ZipEntry; +import java.util.zip.ZipException; +import java.util.zip.ZipFile; + +/// Builds the Unity project of an application module (`src/main/unity`, laid +/// out like any Unity project: `Assets/` and `ProjectSettings/`) into what +/// every Codename One target already knows how to build. Shared by the Maven +/// goal `compile-unity` and the Gradle task. +/// +/// Four steps, each a child process so the build tool's own JVM never loads +/// the translator: +/// +/// 1. the .NET SDK compiles `Assets/**/*.cs` to one assembly, against the +/// reference assemblies of the runtime (never against Unity's own); +/// 2. the CIL translator turns that assembly into class files; +/// 3. the scene compiler turns the scenes and prefabs into Java source +/// (`com.codename1.generated.unity.UnityAppImpl`), because a device build +/// has no reflection to instantiate them with at run time; +/// 4. the classes and the images the scenes draw are installed in the classes +/// directory, where the build tool's own `javac` then compiles the +/// generated source beside them. +/// +/// Steps 1 to 3 are skipped while nothing they read has changed, so a project +/// that is only being run again does not need the .NET SDK at all. Nothing is +/// written outside the build directory: the SDK's first-run files and its +/// package cache are pointed into it unless the environment already names +/// them. +public class UnityProjectBuilder { + + /// The runtime translated scripts run against. An application depends on + /// it like on any library, so every target ships it. + public static final String RUNTIME_ARTIFACT = "codenameone-unity-compat"; + + /// The translator and scene compiler; a build-time tool, never shipped. + public static final String TOOL_ARTIFACT = "codenameone-cil-translator"; + + /// The classifier under which [#RUNTIME_ARTIFACT] publishes the assemblies + /// C# compiles against. A classifier and not resources of the runtime + /// jar, because that jar is merged into the application and the + /// assemblies are megabytes no device has a use for. + public static final String REFERENCES_CLASSIFIER = "references"; + + /// The package of the class the scene compiler generates. + public static final String GENERATED_PACKAGE = "com.codename1.generated.unity"; + + /// The class the scene compiler generates. + public static final String APP_IMPL_CLASS = "UnityAppImpl"; + + /// The lifecycle class a generated main class extends. + public static final String APPLICATION_CLASS = "com.codename1.unitycompat.app.UnityApplication"; + + /// The property both build tools read the `dotnet` executable from. + public static final String DOTNET_PROPERTY = "cn1.unity.dotnet"; + + /// The oldest .NET SDK accepted. The scripts are compiled as C# 9 for + /// `netstandard2.1`; this is the oldest long-term-support SDK whose + /// compiler does that. + static final int MINIMUM_SDK_MAJOR = 6; + + private static final String NETSTANDARD = "netstandard.dll"; + private static final String ENGINE = "UnityEngine.dll"; + private static final String VALUES = "Codename1.UnityValues.dll"; + private static final String SCRIPTS = "Assembly-CSharp.dll"; + private static final String TRANSLATOR_MAIN = "com.codename1.cil.translate.Translator"; + private static final String SCENE_COMPILER_MAIN = "com.codename1.unity.scenecompiler.SceneCompiler"; + /// Bumped when what the steps write changes, so an upgraded plugin does + /// not trust the staging directory an older one left. Also when the + /// state file changes: 3 is the first to record the parts and the + /// entries behind the digest (see [State]). + static final String STATE_VERSION = "3"; + /// The four things the digest is of, as the state file names them and + /// in the order a mismatch is reported. + private static final String[] PARTS = {"project", "runtime", "references", "tool"}; + private static final String RUNTIME_LABEL = "runtime!"; + private static final String REFERENCES_LABEL = "references!"; + private static final String TOOL_LABEL = "tool!"; + private final File unityDir; + private final File javaOut; + private final File classesOut; + private final File workDir; + private final File runtimeJar; + private final File referencesJar; + private final List toolClasspath; + private final String dotnetSetting; + private final String mainPackage; + private final String mainClass; + private final List sourceRoots; + private final Log log; + private Toolchain toolchain; + /// Why an output that was already under the work directory is not being + /// used, or null when there was none. Said along with a missing SDK: a + /// build that was given compiled output and still asks for the SDK was + /// given output of something else, and that is the fact to act on. + private String staleOutput; + /// Why the content the digest is of could not be read, when it could not. + private String unreadable; + + /// The three external steps. An interface so the tests can count and fake + /// them: the logic around them (what is skipped, what is installed, what + /// is removed) must be testable on a machine with no .NET SDK. + interface Toolchain { + /// Compiles the generated C# project; its assembly lands in + /// `bin/Release/netstandard2.1` beside it. + void compileScripts(File csproj, File logFile) throws BuildException; + + /// Translates `assemblies` to class files under `out`. Returns what + /// the translator printed. + List translate(File out, File runtimeClasses, List references, List assemblies, + File logFile) throws BuildException; + + /// Compiles the project's scenes to `javaOut` and copies the images + /// they draw to `resources`. Returns what the compiler printed. + List compileScenes(File project, File javaOut, File resources, List references, + List assemblies, File logFile) throws BuildException; + } + + /// Logs the `warning:` and `note:` lines of a tool's output, and + /// answers how many warnings there were. + private int report(List lines) { + int warnings = 0; + for (String line : lines) { + if (line.startsWith("warning:")) { + log.warn("src/main/unity: " + line.substring("warning:".length()).trim()); + warnings++; + } else if (line.startsWith("note:")) { + log.info("src/main/unity: " + line.substring("note:".length()).trim()); + } + } + return warnings; + } + + /// @param unityDir `src/main/unity` + /// @param javaOut generated sources (`target/generated-sources/unity`) + /// @param classesOut where the translated classes and the images go (the classes directory) + /// @param buildDir `target`; everything else is written under its `unity` directory + /// @param runtimeJar the `codenameone-unity-compat` jar + /// @param referencesJar its `references` classifier + /// @param toolClasspath the `codenameone-cil-translator` jar and its dependencies + /// @param dotnetSetting the `dotnet` executable or its directory, or null to look for it + /// @param mainPackage `codename1.packageName`, or null + /// @param mainClass `codename1.mainName`, or null + /// @param sourceRoots the project's own source roots, to see whether the main class exists + public UnityProjectBuilder(File unityDir, File javaOut, File classesOut, File buildDir, File runtimeJar, + File referencesJar, List toolClasspath, String dotnetSetting, + String mainPackage, String mainClass, List sourceRoots, Log log) { + this.unityDir = unityDir; + this.javaOut = javaOut; + this.classesOut = classesOut; + this.workDir = new File(buildDir, "unity"); + this.runtimeJar = runtimeJar; + this.referencesJar = referencesJar; + this.toolClasspath = toolClasspath == null ? new ArrayList() : new ArrayList(toolClasspath); + this.dotnetSetting = dotnetSetting; + this.mainPackage = mainPackage; + this.mainClass = mainClass; + this.sourceRoots = sourceRoots == null ? new ArrayList() : new ArrayList(sourceRoots); + this.log = log; + } + + /// A Unity project has both directories. `Assets` alone is not enough to + /// go by: the scenes to build, the physics settings and the input axes + /// are all read from `ProjectSettings`. + public static boolean isUnityProject(File unityDir) { + return unityDir != null && new File(unityDir, "Assets").isDirectory() + && new File(unityDir, "ProjectSettings").isDirectory(); + } + + void setToolchain(Toolchain toolchain) { + this.toolchain = toolchain; + } + + /// Builds the project unless nothing changed since the last run. Returns + /// false when there is no Unity project, after removing what a build of + /// one that used to be there left behind (see [#removeOutput()]). + public boolean run() throws BuildException { + if (!isUnityProject(unityDir)) { + if (unityDir != null && unityDir.isDirectory()) { + log.warn(unityDir + " is not a Unity project (it needs both Assets and ProjectSettings); ignored"); + } + removeOutput(); + return false; + } + if (runtimeJar == null || !runtimeJar.isFile()) { + throw new BuildException("src/main/unity exists but the " + RUNTIME_ARTIFACT + + " dependency is missing; add it to the common module, or run cn1:import-unity-project," + + " which adds it."); + } + if (referencesJar == null || !referencesJar.isFile()) { + throw new BuildException("The reference assemblies of " + RUNTIME_ARTIFACT + " (its '" + + REFERENCES_CLASSIFIER + "' classifier) could not be resolved; the C# scripts in" + + " src/main/unity cannot be compiled without them."); + } + File state = new File(workDir, "state.txt"); + File staged = new File(workDir, "app"); + File resources = new File(workDir, "resources"); + File runtimeClasses = new File(workDir, "runtime"); + File appImpl = new File(javaOut, GENERATED_PACKAGE.replace('.', File.separatorChar) + File.separator + + APP_IMPL_CLASS + ".java"); + State now = currentState(); + State recorded = State.parse(read(state)); + // No digest, no skipping: content that could not be read matches + // nothing, not even a state file that is empty for the same reason. + String missing = missingOutput(staged, appImpl, resources, runtimeClasses); + boolean compiled = false; + if (now != null && recorded != null && now.digest.equals(recorded.digest) && missing == null) { + log.debug("Unity project unchanged; not compiled again"); + } else { + staleOutput = staleOutput(state, recorded, now, missing); + if (staleOutput != null) { + log.info(staleOutput); + } + // Deleted first: a step that fails must not leave the next build + // believing the staging directory matches the sources. + deleteQuietly(state); + compile(staged, resources, runtimeClasses); + if (!appImpl.isFile()) { + throw new BuildException("The scene compiler wrote no " + appImpl + " (log: " + + new File(workDir, "scene-compiler.log") + ")"); + } + compiled = true; + } + install(staged, resources, runtimeClasses, compiled); + // Recorded only once installed: an install that failed part way must + // be repeated in full by the next build, and only a build that + // compiles replaces every output (see install). + if (compiled && now != null) { + write(state, now.text()); + } + writeMainClass(); + return true; + } + + /// Removes what a build of a Unity project left behind, for a module + /// whose project is gone -- `src/main/unity` deleted, or one of its two + /// directories. Returns whether there was anything to remove. + /// + /// Without this the output outlives its sources until a clean build. The + /// translated classes and the images are in the classes directory, where + /// no compiler put them and so none takes them out again, and every + /// target's packaging reads that directory: a game deleted from the + /// sources would go on being shipped. The generated scenes would go on + /// being compiled wherever the directory is a source root. + /// + /// Only what a build of ours wrote is touched, by the same record + /// `install` removes a deleted script's class by: the classes directory + /// is shared with javac, and the generated sources directory may be one + /// the developer configured. A module that never had a Unity project is + /// answered from three file checks, and nothing is created for it. + public boolean removeOutput() { + String generatedPackage = GENERATED_PACKAGE.replace('.', File.separatorChar); + File scenes = new File(javaOut, generatedPackage); + boolean named = mainPackage != null && mainClass != null && mainPackage.length() > 0 + && mainClass.length() > 0; + File main = named ? generatedMainClass() : null; + boolean mainGenerated = main != null && main.isFile(); + if (!workDir.exists() && !scenes.exists() && !mainGenerated) { + return false; + } + String recorded = read(new File(workDir, "installed.txt")); + if (recorded != null) { + for (String rel : recorded.split("\n")) { + // A record is only ever written by install(), but it is a file + // in the build directory: a line that climbs out of the classes + // directory is not followed. + if (rel.length() > 0 && !rel.contains("..")) { + deleteAndPrune(new File(classesOut, rel), classesOut); + } + } + } + // What javac made of the generated scenes. In a build that keeps the + // translated classes in a directory of their own there is none here. + File sceneClasses = new File(classesOut, generatedPackage); + delete(sceneClasses); + prune(sceneClasses.getParentFile(), classesOut); + delete(scenes); + prune(scenes.getParentFile(), javaOut); + if (mainGenerated) { + // A main class in the generated sources is ours: the build deletes + // it as soon as the application has its own. Its class goes with + // it, or the application would still start a game it no longer has + // -- and fail on the generated class that was just removed. + deleteAndPrune(main, javaOut); + deleteAndPrune(new File(classesOut, mainPackage.replace('.', File.separatorChar) + File.separator + + mainClass + ".class"), classesOut); + } + // The state file with everything else: a project that comes back is + // compiled again, not matched against a staging directory of before. + delete(workDir); + if (log != null) { + log.info("There is no Unity project in " + unityDir + " any more; removed what an earlier build of it" + + " left in " + classesOut + " and " + javaOut); + } + return true; + } + + private static void deleteAndPrune(File f, File root) { + deleteQuietly(f); + prune(f.getParentFile(), root); + } + + /// Deletes `dir` and its parents while they are empty, up to and not + /// including `root`: a directory left in the classes directory is an + /// entry in the application's jar. + private static void prune(File dir, File root) { + File d = dir; + while (d != null && !d.equals(root) && d.toPath().startsWith(root.toPath())) { + String[] children = d.list(); + if (children == null || children.length > 0) { + return; + } + deleteQuietly(d); + d = d.getParentFile(); + } + } + + /// The first thing a full build would have left under the work directory + /// that is not there, or null when it is all there. A matching state file + /// says the output was made from these sources, not that it is still + /// whole: a cache restored in part, or a directory deleted by hand, and + /// [#install] would put an application together from what is left. + /// + /// Every compile creates the staging and image directories and unpacks + /// the runtime's classes, so all three exist after one -- the images + /// directory empty for a project that draws none, which is why it is + /// asked to exist and not to hold something. The runtime's directory is + /// never empty: install skips each staged class the runtime ships by + /// looking there, and an empty one installs a second copy of each. And + /// every file the last install copied (`installed.txt`) must still be + /// where it was copied from, which is what notices an image or a class + /// gone from a directory that is itself there. + private String missingOutput(File staged, File appImpl, File resources, File runtimeClasses) { + if (!staged.isDirectory()) { + return staged.toString(); + } + if (!appImpl.isFile()) { + return appImpl.toString(); + } + if (!resources.isDirectory()) { + return resources.toString(); + } + String[] runtime = runtimeClasses.list(); + if (runtime == null || runtime.length == 0) { + return runtimeClasses.toString(); + } + String recorded = read(new File(workDir, "installed.txt")); + if (recorded != null) { + for (String rel : recorded.split("\n")) { + if (rel.length() > 0 && !new File(staged, rel).isFile() && !new File(resources, rel).isFile()) { + return new File(resources, rel) + " (or " + new File(staged, rel) + ")"; + } + } + } + return null; + } + + /// What to say about compiled output that is present and cannot be used, + /// or null when the work directory holds none: a first build has nothing + /// to explain. + private String staleOutput(File state, State recorded, State now, String missing) { + if (recorded == null) { + return null; + } + if (missing != null) { + return state + " is there, but " + missing + " is not: the" + + " compiled output of " + unityDir + " is incomplete and has to be made again."; + } + return "The compiled output under " + workDir + " does not match these sources or jars: " + state + + " records other content than " + unityDir + ", the " + RUNTIME_ARTIFACT + " jar, its '" + + REFERENCES_CLASSIFIER + "' jar and the " + TOOL_ARTIFACT + " class path have now. " + + (now == null ? "What is there now could not be read: " + unreadable + "." + : differences(recorded, now)) + + " Output compiled elsewhere is good only with the exact project files and jars it was" + + " compiled from, so the project has to be compiled again."; + } + + /// The state file: the digest the build is skipped on, and under it what + /// the digest is of -- a digest of each of the four parts, then every + /// entry of every part. Only the first line decides anything. The rest is + /// there for the build that finds another digest, which can then say + /// which part differs and name a file in it; a digest alone says only + /// that something does, and the machine that made the output is not the + /// one being asked. + /// + /// ``` + /// + /// version=3 + /// part.project= + /// part.runtime=... + /// part.references=... + /// part.tool=... + /// Assets/Scripts/Player.cs= + /// runtime!com/acme/A.class:: + /// ``` + static final class State { + final String digest; + /// Null in a file with no `version=` line: one an older plugin + /// wrote, or one that was not written by a build at all. + final String version; + /// A digest for each of [#PARTS] that is recorded. + final Map parts; + /// Sorted as `String` compares them. + final List entries; + + State(String digest, String version, Map parts, List entries) { + this.digest = digest; + this.version = version; + this.parts = parts; + this.entries = entries; + } + + /// The state of sorted `entries`. + static State of(List entries) throws NoSuchAlgorithmException { + MessageDigest md = MessageDigest.getInstance("SHA-256"); + Map parts = new LinkedHashMap(); + for (String part : PARTS) { + md.reset(); + for (String e : entries) { + if (part.equals(partOf(e))) { + md.update((byte) '\n'); + md.update(e.getBytes(StandardCharsets.UTF_8)); + } + } + StringBuilder sb = new StringBuilder(); + hex(sb, md.digest()); + parts.put(part, sb.toString()); + } + md.reset(); + md.update(STATE_VERSION.getBytes(StandardCharsets.UTF_8)); + for (String e : entries) { + md.update((byte) '\n'); + md.update(e.getBytes(StandardCharsets.UTF_8)); + } + StringBuilder sb = new StringBuilder(); + hex(sb, md.digest()); + return new State(sb.toString(), STATE_VERSION, parts, entries); + } + + /// What a state file holds, or null for no file. Anything is read: + /// a file of one line is a digest with nothing recorded under it. + static State parse(String text) { + if (text == null) { + return null; + } + String digest = null; + String version = null; + Map parts = new LinkedHashMap(); + List entries = new ArrayList(); + int start = 0; + int length = text.length(); + while (start <= length) { + int end = text.indexOf('\n', start); + if (end < 0) { + end = length; + } + String line = text.substring(start, end); + if (line.endsWith("\r")) { + line = line.substring(0, line.length() - 1); + } + start = end + 1; + if (digest == null) { + digest = line.trim(); + } else if (line.startsWith("version=")) { + version = line.substring("version=".length()); + } else if (line.startsWith("part.") && line.indexOf('=') > 0) { + int eq = line.indexOf('='); + parts.put(line.substring("part.".length(), eq), line.substring(eq + 1)); + } else if (line.length() > 0) { + entries.add(line); + } + } + return new State(digest == null ? "" : digest, version, parts, entries); + } + + String text() { + StringBuilder sb = new StringBuilder(digest).append('\n'); + sb.append("version=").append(version).append('\n'); + for (Map.Entry part : parts.entrySet()) { + sb.append("part.").append(part.getKey()).append('=').append(part.getValue()).append('\n'); + } + for (String e : entries) { + sb.append(e).append('\n'); + } + return sb.toString(); + } + } + + /// Which of [#PARTS] an entry belongs to. A file of the project is its + /// path, which starts with `Assets/` or `ProjectSettings/`; the entries + /// of the three jars start with the label of their role. + private static String partOf(String entry) { + if (entry.startsWith(RUNTIME_LABEL)) { + return PARTS[1]; + } + if (entry.startsWith(REFERENCES_LABEL)) { + return PARTS[2]; + } + if (entry.startsWith(TOOL_LABEL)) { + return PARTS[3]; + } + return PARTS[0]; + } + + /// The name in an entry, without the digest or the length and checksum + /// after it: `Assets/a.cs=` and `tool!a/B.class::`. + private static String nameOf(String entry) { + String part = partOf(entry); + if (PARTS[0].equals(part)) { + int eq = entry.lastIndexOf('='); + return eq < 0 ? entry : entry.substring(0, eq); + } + String name = entry.substring(entry.indexOf('!') + 1); + int crc = name.lastIndexOf(':'); + int size = crc < 0 ? -1 : name.lastIndexOf(':', crc - 1); + if (size >= 0) { + return name.substring(0, size); + } + // A file on the class path that is no jar has no name here, only + // the digest of its bytes. + return "a file that is not a jar"; + } + + private static String describe(String part) { + if (PARTS[0].equals(part)) { + return "the project files"; + } + if (PARTS[1].equals(part)) { + return "the " + RUNTIME_ARTIFACT + " jar"; + } + if (PARTS[2].equals(part)) { + return "the '" + REFERENCES_CLASSIFIER + "' jar"; + } + return "the " + TOOL_ARTIFACT + " class path"; + } + + /// One sentence on what differs between the state a file records and + /// the state found now: each part whose digest is another, with how many + /// of its entries differ and the first of them. Without it a mismatch + /// says that one of four things is not what it was, on a machine that + /// cannot be looked at afterwards. + static String differences(State recorded, State now) { + if (recorded.version == null || recorded.parts.isEmpty()) { + return "Which of them differs cannot be said: the file records a digest (" + + abbreviate(recorded.digest) + ", now " + abbreviate(now.digest) + ") and nothing under" + + " it, as one written by an older build or by hand does."; + } + StringBuilder sb = new StringBuilder(); + for (String part : PARTS) { + String was = recorded.parts.get(part); + String is = now.parts.get(part); + if (was != null && was.equals(is)) { + continue; + } + sb.append(sb.length() == 0 ? "What differs: " : "; ").append(describe(part)); + sb.append(" (").append(abbreviate(was)).append(" recorded, ").append(abbreviate(is)).append(" now"); + describeEntries(sb, part, recorded.entries, now.entries); + sb.append(')'); + } + if (sb.length() == 0) { + // Every part is what it was, so the content is: the digest also + // covers the version of this record. + return "All four are the same content: the file was written as version " + recorded.version + + " of this record and the build writes version " + now.version + "."; + } + return sb.append('.').toString(); + } + + /// Appends how many entries of `part` differ and what the first one is. + /// Both lists are sorted, so one pass over the two finds them in order. + private static void describeEntries(StringBuilder sb, String part, List recorded, List now) { + List was = entriesOf(part, recorded); + List is = entriesOf(part, now); + if (was.isEmpty()) { + sb.append("; the file lists none of its ").append(is.size()).append(" entries"); + return; + } + int i = 0; + int j = 0; + int count = 0; + String first = null; + while (i < was.size() || j < is.size()) { + String a = i < was.size() ? was.get(i) : null; + String b = j < is.size() ? is.get(j) : null; + if (a != null && a.equals(b)) { + i++; + j++; + continue; + } + String nameA = a == null ? null : nameOf(a); + String nameB = b == null ? null : nameOf(b); + int order = nameA == null ? 1 : nameB == null ? -1 : nameA.compareTo(nameB); + String what; + if (order == 0) { + what = nameA + " has other content"; + i++; + j++; + } else if (order < 0) { + what = nameA + " is recorded and is not there now"; + i++; + } else { + what = nameB + " is there now and is not recorded"; + j++; + } + if (first == null) { + first = what; + } + count++; + } + sb.append("; ").append(count).append(" of ").append(Math.max(was.size(), is.size())) + .append(" entries differ, the first: ").append(first); + } + + private static List entriesOf(String part, List entries) { + List out = new ArrayList(); + for (String e : entries) { + if (part.equals(partOf(e))) { + out.add(e); + } + } + return out; + } + + /// Enough of a digest to tell two apart in a log line. + private static String abbreviate(String digest) { + if (digest == null || digest.length() == 0) { + return "none"; + } + return digest.length() <= 12 ? digest : digest.substring(0, 12); + } + + private void compile(File staged, File resources, File runtimeClasses) throws BuildException { + if (toolClasspath.isEmpty() && toolchain == null) { + throw new BuildException("The " + TOOL_ARTIFACT + " tool could not be resolved; the Unity project in" + + " src/main/unity cannot be translated without it."); + } + Toolchain tools = toolchain != null ? toolchain : new ForkedToolchain(); + File refs = new File(workDir, "ref"); + File cs = new File(workDir, "cs"); + delete(refs); + delete(cs); + delete(staged); + delete(resources); + delete(runtimeClasses); + // The scene compiler's output is the only thing of ours in javaOut + // that a changed project can make stale. + delete(new File(javaOut, GENERATED_PACKAGE.replace('.', File.separatorChar))); + mkdirs(refs); + mkdirs(cs); + mkdirs(staged); + mkdirs(resources); + mkdirs(javaOut); + try { + extract(referencesJar, refs, ".dll"); + extract(runtimeJar, runtimeClasses, ".class"); + } catch (IOException e) { + throw new BuildException("Cannot unpack the Unity runtime: " + e.getMessage(), e); + } + File netstandard = new File(refs, NETSTANDARD); + File engine = new File(refs, ENGINE); + File values = new File(refs, VALUES); + for (File f : new File[] {netstandard, engine, values}) { + if (!f.isFile()) { + throw new BuildException(referencesJar + " has no " + f.getName() + + "; it is not the reference assemblies of " + RUNTIME_ARTIFACT); + } + } + File csproj = new File(cs, "Assembly-CSharp.csproj"); + write(csproj, csproj(new File(unityDir, "Assets"), netstandard, engine, values)); + + log.info("Compiling the C# scripts of " + unityDir); + tools.compileScripts(csproj, new File(workDir, "dotnet.log")); + File scripts = new File(cs, "bin/Release/netstandard2.1/" + SCRIPTS); + if (!scripts.isFile()) { + throw new BuildException("The C# build produced no " + scripts + " (log: " + + new File(workDir, "dotnet.log") + ")"); + } + List references = new ArrayList(); + references.add(engine); + references.add(netstandard); + // The value types are translated along with the scripts, not only + // referenced: the translator keeps the names of what it translates and + // maps what it merely references onto the hand-written runtime. + List assemblies = new ArrayList(); + assemblies.add(values); + assemblies.add(scripts); + // Both tools say what they could not carry over, a line each; a + // build that kept those to its log files would be a game that + // differs from the one in Unity with nothing to say why. + int warnings = report(tools.translate(staged, runtimeClasses, references, assemblies, + new File(workDir, "translate.log"))); + List sceneAssemblies = new ArrayList(); + sceneAssemblies.add(scripts); + sceneAssemblies.add(values); + warnings += report(tools.compileScenes(unityDir, javaOut, resources, references, sceneAssemblies, + new File(workDir, "scene-compiler.log"))); + log.info("Compiled the Unity project" + (warnings == 0 ? "" : " with " + warnings + + " warning(s): what they name will not behave as it does in Unity")); + } + + /// The C# project that compiles the scripts where they are. Its + /// references are the three assemblies and nothing else: with the + /// implicit framework reference left on, the SDK would download the + /// `netstandard` targeting pack from NuGet on every clean build, and fail + /// offline. + static String csproj(File assets, File netstandard, File engine, File values) { + String a = msbuild(assets.getAbsolutePath()); + return "\n" + + " \n" + + " Library\n" + + " netstandard2.1\n" + + " 9.0\n" + + " disable\n" + + " disable\n" + + " Assembly-CSharp\n" + + " true\n" + + " portable\n" + + " true\n" + + " false\n" + + " false\n" + + " true\n" + + " $(NoWarn);CS0649;CS0414;CS0169;CS0108;CS0114;CS0618\n" + + " \n" + + " \n" + // Editor scripts are not part of a player build. + + " \n" + + reference("netstandard", netstandard) + + reference("UnityEngine", engine) + + reference("Codename1.UnityValues", values) + + " \n" + + "\n"; + } + + private static String reference(String name, File assembly) { + return " \n" + + " " + msbuild(assembly.getAbsolutePath()) + "\n" + + " false\n" + + " \n"; + } + + /// A path as MSBuild reads it literally, inside XML: its own special + /// characters percent-encoded, then XML's escaped. + static String msbuild(String path) { + StringBuilder b = new StringBuilder(); + for (int i = 0; i < path.length(); i++) { + char c = path.charAt(i); + switch (c) { + case '%': + b.append("%25"); + break; + case '$': + b.append("%24"); + break; + case '@': + b.append("%40"); + break; + case ';': + b.append("%3B"); + break; + case '\'': + b.append("%27"); + break; + case '*': + b.append("%2A"); + break; + case '?': + b.append("%3F"); + break; + case '&': + b.append("&"); + break; + case '<': + b.append("<"); + break; + case '>': + b.append(">"); + break; + case '"': + b.append("""); + break; + case '\\': + b.append('/'); + break; + default: + b.append(c); + break; + } + } + return b.toString(); + } + + /// Copies what the steps staged into the classes directory, and removes + /// what an earlier run installed and this one did not: a script deleted + /// from the project must not stay behind as a class. + /// + /// This runs on every build, also when nothing was compiled: the classes + /// directory is the build tool's, and a clean of it alone must not lose + /// the game. + /// + /// When `compiled`, the steps have just written every staged file, and + /// each one replaces the installed copy whatever its size and time. A + /// changed class or image can keep its length, and be written within + /// the same tick of the file system's clock as the copy it replaces -- + /// or after a copy whose time is ahead -- so length and time alone + /// would keep the stale one. When nothing was compiled, the staged + /// files are the ones installed last time, and length and time are + /// enough to refill a cleaned directory without copying the rest. + private void install(File staged, File resources, File runtimeClasses, boolean compiled) + throws BuildException { + File record = new File(workDir, "installed.txt"); + Set previous = new LinkedHashSet(); + String recorded = read(record); + if (recorded != null) { + for (String line : recorded.split("\n")) { + if (line.length() > 0) { + previous.add(line); + } + } + } + Set installed = new TreeSet(); + List files = new ArrayList(); + collect(staged, "", files); + int classes = 0; + try { + for (String rel : files) { + // The runtime jar already ships the value types, translated + // from the same assembly. A second copy in the application + // would be one more definition of the class for each target's + // packaging to pick between. + if (new File(runtimeClasses, rel).isFile()) { + continue; + } + install(new File(staged, rel), new File(classesOut, rel), compiled); + installed.add(rel); + classes++; + } + files.clear(); + collect(resources, "", files); + for (String rel : files) { + install(new File(resources, rel), new File(classesOut, rel), compiled); + installed.add(rel); + } + } catch (IOException e) { + throw new BuildException("Cannot install the Unity project into " + classesOut + ": " + e.getMessage(), e); + } + for (String rel : previous) { + if (!installed.contains(rel)) { + deleteQuietly(new File(classesOut, rel)); + } + } + StringBuilder sb = new StringBuilder(); + for (String rel : installed) { + sb.append(rel).append('\n'); + } + write(record, sb.toString()); + log.debug("Installed " + classes + " translated classes and " + files.size() + " images in " + classesOut); + } + + private static void install(File from, File to, boolean always) throws IOException { + if (!always && to.isFile() && to.length() == from.length() && to.lastModified() >= from.lastModified()) { + return; + } + File parent = to.getParentFile(); + if (parent != null) { + mkdirs(parent); + } + Files.copy(from.toPath(), to.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + + private File generatedMainClass() { + return new File(javaOut, mainPackage.replace('.', File.separatorChar) + File.separator + mainClass + ".java"); + } + + /// True when the application has its own main class, in which case the + /// generated one is deleted: left in the generated sources next to the + /// developer's class, javac would see two definitions until a clean build. + private boolean removeGeneratedMainClassIfWritten() { + String rel = mainPackage.replace('.', File.separatorChar) + File.separator + mainClass; + for (File root : sourceRoots) { + if (root.equals(javaOut)) { + continue; + } + if (new File(root, rel + ".java").isFile() || new File(root, rel + ".kt").isFile()) { + File generated = generatedMainClass(); + if (generated.isFile() && !generated.delete()) { + log.warn("Cannot delete the generated " + generated + "; the application now has its own"); + } + return true; + } + } + return false; + } + + /// Generates the Codename One main class when the project does not have + /// one, so a Unity project builds without any Codename One code of its + /// own. Checked on every run and not only when the project changed: a + /// main class written or deleted since the last build must be noticed. + private void writeMainClass() throws BuildException { + if (mainPackage == null || mainClass == null || mainPackage.length() == 0 || mainClass.length() == 0) { + return; + } + if (removeGeneratedMainClassIfWritten()) { + return; + } + File out = generatedMainClass(); + String src = mainClassSource(mainPackage, mainClass, + "// Generated by the Codename One Unity build: the application's Codename One\n" + + "// entry point. Write this class yourself to customize startup.\n", ""); + if (src.equals(read(out))) { + return; + } + write(out, src); + } + + /// The source of a main class that runs the project's first scene. The + /// subclass names the generated class itself, as source: the runtime jar + /// was compiled before the project existed and has no other way to reach + /// it without reflection. + static String mainClassSource(String pkg, String cls, String fileComment, String classComment) { + return fileComment + + "package " + pkg + ";\n\n" + + classComment + + "public class " + cls + " extends " + APPLICATION_CLASS + " {\n" + + " @Override\n" + + " protected void installProject() {\n" + + " " + GENERATED_PACKAGE + "." + APP_IMPL_CLASS + ".install();\n" + + " }\n" + + "}\n"; + } + + /// What the build is skipped on the strength of: the content of + /// everything the steps read, and nothing of where or when. + /// + /// The staging directory may be made on one machine and used on + /// another -- compiled once where the .NET SDK is, then handed with its + /// state file to the jobs that package each target -- and a fresh + /// checkout there has its own path and its own modification times. So + /// neither is in here. Every file of `Assets` and `ProjectSettings` is + /// its path below the project, with `/` between the names, and a + /// SHA-256 of its bytes; the lines are sorted as `String` compares + /// them, which no locale changes. A length and a time would also miss + /// an edit of the same length whose time was kept, as a checkout that + /// restores times makes of `speed = 1` to `speed = 2`. + /// + /// That reads the whole project on every build, also the ones that + /// change nothing. Measured: SHA-256 through 256 MB took 1.1 to 1.6 + /// seconds on JDK 8 and on JDK 17 here, about 200 MB a second, so a + /// gigabyte of pictures and sound is some five seconds; the sample + /// projects under `scripts/unity-compat-samples`, 0.2 MB between + /// them, take under ten milliseconds. + /// + /// The runtime, the references and the tool are in it by what their + /// jars hold -- see [#stamp] -- and by the role they have here, never + /// by a file name, which has a version and a directory in it. + /// + /// Returns null when the content could not be read, and leaves why in + /// [#unreadable]: no digest, no skipping. + private State currentState() { + try { + MessageDigest md = MessageDigest.getInstance("SHA-256"); + byte[] buffer = new byte[1 << 16]; + List entries = new ArrayList(); + collectStamped(new File(unityDir, "Assets"), "Assets/", entries, md, buffer); + collectStamped(new File(unityDir, "ProjectSettings"), "ProjectSettings/", entries, md, buffer); + stamp(runtimeJar, RUNTIME_LABEL, entries, md, buffer); + stamp(referencesJar, REFERENCES_LABEL, entries, md, buffer); + for (File f : toolClasspath) { + // One label for all of them: the order of a class path and + // how many jars the classes came in are not content. + stamp(f, TOOL_LABEL, entries, md, buffer); + } + Collections.sort(entries); + return State.of(entries); + } catch (NoSuchAlgorithmException e) { + unreadable = e.toString(); + return null; + } catch (IOException e) { + // A file that cannot be read: said with the mismatch, since the + // build that follows may fail on something else first. + unreadable = e.toString(); + return null; + } + } + + /// The lines of one jar, or of a directory of classes standing in for + /// one. A jar is read by its directory: each entry's name, length and + /// CRC-32, which is of the entry's bytes as they are unpacked. The file + /// itself is not hashed, since the same classes packed twice are two + /// different files -- every entry carries the time it was packed at. + /// `META-INF` is left out for the same reason: the manifest names the + /// JDK and the machine that packed it, and no step reads anything + /// there. + private static void stamp(File f, String label, List out, MessageDigest md, byte[] buffer) + throws IOException { + if (f.isDirectory()) { + collectStamped(f, label, out, md, buffer); + return; + } + try (ZipFile zip = new ZipFile(f)) { + Enumeration entries = zip.entries(); + while (entries.hasMoreElements()) { + ZipEntry e = entries.nextElement(); + if (!e.isDirectory() && !e.getName().startsWith("META-INF/")) { + out.add(label + e.getName() + ":" + e.getSize() + ":" + Long.toHexString(e.getCrc())); + } + } + } catch (ZipException e) { + // Not a jar: a file of some other kind on the class path, by + // its bytes and without its name. + md.reset(); + try (InputStream in = Files.newInputStream(f.toPath())) { + int n = in.read(buffer); + while (n >= 0) { + md.update(buffer, 0, n); + n = in.read(buffer); + } + } + StringBuilder line = new StringBuilder(label).append('='); + hex(line, md.digest()); + out.add(line.toString()); + } + } + + /// One line for each file below a directory: its path from `prefix` + /// on, and a SHA-256 of its bytes. + private static void collectStamped(File dir, String prefix, List out, MessageDigest md, byte[] buffer) + throws IOException { + File[] files = dir.listFiles(); + if (files == null) { + return; + } + for (File f : files) { + String name = f.getName(); + if (f.isDirectory()) { + collectStamped(f, prefix + name + "/", out, md, buffer); + continue; + } + md.reset(); + try (InputStream in = Files.newInputStream(f.toPath())) { + int n = in.read(buffer); + while (n >= 0) { + md.update(buffer, 0, n); + n = in.read(buffer); + } + } + StringBuilder line = new StringBuilder(prefix).append(name).append('='); + hex(line, md.digest()); + out.add(line.toString()); + } + } + + private static void hex(StringBuilder sb, byte[] bytes) { + for (byte b : bytes) { + sb.append(Integer.toHexString((b & 0xff) | 0x100).substring(1)); + } + } + + private static void collect(File dir, String prefix, List out) { + File[] files = dir.listFiles(); + if (files == null) { + return; + } + for (File f : files) { + if (f.isDirectory()) { + collect(f, prefix + f.getName() + "/", out); + } else { + out.add(prefix + f.getName()); + } + } + } + + /// Unpacks the entries of `jar` whose names end in `suffix`. + private static void extract(File jar, File into, String suffix) throws IOException { + String root = into.getCanonicalPath() + File.separator; + try (ZipFile zip = new ZipFile(jar)) { + Enumeration entries = zip.entries(); + while (entries.hasMoreElements()) { + ZipEntry e = entries.nextElement(); + if (e.isDirectory() || !e.getName().endsWith(suffix)) { + continue; + } + File out = new File(into, e.getName()); + // An entry named with ".." would be written outside the + // directory; no jar of ours has one. + if (!out.getCanonicalPath().startsWith(root)) { + throw new IOException(jar + " has an entry outside its root: " + e.getName()); + } + File parent = out.getParentFile(); + if (parent != null) { + mkdirs(parent); + } + try (InputStream in = zip.getInputStream(e)) { + Files.copy(in, out.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + } + } + } + + private static void mkdirs(File dir) { + if (!dir.isDirectory() && !dir.mkdirs() && !dir.isDirectory()) { + throw new IllegalStateException("Cannot create " + dir); + } + } + + private static void delete(File f) { + File[] children = f.listFiles(); + if (children != null) { + for (File c : children) { + delete(c); + } + } + deleteQuietly(f); + } + + private static void deleteQuietly(File f) { + try { + Files.deleteIfExists(f.toPath()); + } catch (IOException e) { + // Whatever is in the way is reported by the step that then fails + // to write here, with the path. + return; + } + } + + /// The text of a file, or null when it cannot be read. + private static String read(File f) { + if (!f.isFile()) { + return null; + } + try { + return new String(Files.readAllBytes(f.toPath()), StandardCharsets.UTF_8); + } catch (IOException e) { + return null; + } + } + + private static void write(File f, String text) throws BuildException { + try { + File parent = f.getParentFile(); + if (parent != null) { + mkdirs(parent); + } + Files.write(f.toPath(), text.getBytes(StandardCharsets.UTF_8)); + } catch (IOException e) { + throw new BuildException("Cannot write " + f + ": " + e.getMessage(), e); + } + } + + // ------------------------------------------------------------- the SDK + + static boolean isWindows() { + return File.separatorChar == '\\'; + } + + /// The `dotnet` executable: the explicit setting, then `DOTNET_ROOT`, then + /// the `PATH`. The setting may name the executable, the directory it is + /// in, or a bare command to look for on the `PATH`. + /// + /// @param setting the value of [#DOTNET_PROPERTY], or null + /// @param env the environment to read `DOTNET_ROOT` and `PATH` from + static File locateDotnet(String setting, Map env) throws BuildException { + return locateDotnet(setting, env, null); + } + + /// As [#locateDotnet(String, Map)], naming the project that needs the + /// SDK in the failure: a build may be configured to read its Unity + /// project from anywhere, and `src/main/unity` is then a directory that + /// does not exist. + /// + /// @param project the Unity project, or null when it is not known + static File locateDotnet(String setting, Map env, File project) throws BuildException { + String exe = isWindows() ? "dotnet.exe" : "dotnet"; + String path = env.get("PATH"); + if (path == null) { + // Windows spells it Path, and its environment is case insensitive + // to everything but a Java map. + path = env.get("Path"); + } + if (setting != null && setting.trim().length() > 0) { + File f = new File(setting.trim()); + if (f.isDirectory() && new File(f, exe).isFile()) { + return new File(f, exe); + } + if (f.isFile()) { + return f; + } + File found = f.getParent() == null ? onPath(f.getName(), path) : null; + if (found != null) { + return found; + } + throw new BuildException(DOTNET_PROPERTY + " names " + setting + ", which is neither the dotnet" + + " executable nor the directory it is in. " + installHint()); + } + String root = env.get("DOTNET_ROOT"); + if (root != null && root.length() > 0 && new File(root, exe).isFile()) { + return new File(root, exe); + } + File found = onPath(exe, path); + if (found != null) { + return found; + } + throw new BuildException((project == null ? "src/main/unity" : project.getPath()) + + " holds a Unity project, and compiling its C# scripts needs the" + + " .NET SDK, which was not found. Looked at the " + DOTNET_PROPERTY + " property (not set), the" + + " DOTNET_ROOT environment variable (" + (root == null || root.length() == 0 ? "not set" : root) + + ") and the PATH. " + installHint()); + } + + private static String installHint() { + return "Install the .NET SDK " + MINIMUM_SDK_MAJOR + " or newer from https://dotnet.microsoft.com/download" + + " and either put dotnet on the PATH or pass -D" + DOTNET_PROPERTY + "=/path/to/dotnet."; + } + + private static File onPath(String name, String path) { + if (path == null) { + return null; + } + for (String dir : path.split(File.pathSeparator)) { + if (dir.length() == 0) { + continue; + } + File f = new File(dir, name); + if (f.isFile()) { + return f; + } + } + return null; + } + + /// The major version in what `dotnet --version` printed, or -1. The + /// version is the first line that starts with a number; an SDK that has + /// something to say first (a first-run notice) says it on other lines. + static int sdkMajor(List output) { + for (String line : output) { + String t = line.trim(); + int dot = t.indexOf('.'); + if (dot <= 0) { + continue; + } + int major = 0; + boolean digits = true; + for (int i = 0; i < dot; i++) { + char c = t.charAt(i); + if (c < '0' || c > '9' || i > 4) { + digits = false; + break; + } + major = major * 10 + (c - '0'); + } + if (digits) { + return major; + } + } + return -1; + } + + /// The forked implementation: the .NET CLI and this JVM's own `java`. + private final class ForkedToolchain implements Toolchain { + private File dotnet() throws BuildException { + File dotnet; + try { + dotnet = locateDotnet(dotnetSetting, System.getenv(), unityDir); + } catch (BuildException e) { + if (staleOutput == null) { + throw e; + } + throw new BuildException(e.getMessage() + "\n" + staleOutput, e); + } + File logFile = new File(workDir, "dotnet-version.log"); + List cmd = new ArrayList(); + cmd.add(dotnet.getAbsolutePath()); + cmd.add("--version"); + int exit = exec(cmd, logFile, true); + List out = lines(logFile); + int major = sdkMajor(out); + if (exit != 0 || major < 0) { + // The host without an SDK answers --version with an error: + // a runtime-only installation is the usual way to get here. + throw new BuildException(dotnet + " did not report an SDK version (exit code " + exit + "): " + + tail(out, 5) + "\nA .NET runtime alone cannot compile C#. " + installHint()); + } + if (major < MINIMUM_SDK_MAJOR) { + throw new BuildException(dotnet + " is the .NET SDK " + out.get(0).trim() + ", which is too old to" + + " compile the scripts in src/main/unity. " + installHint()); + } + return dotnet; + } + + public void compileScripts(File csproj, File logFile) throws BuildException { + List cmd = new ArrayList(); + cmd.add(dotnet().getAbsolutePath()); + cmd.add("build"); + cmd.add("-c"); + cmd.add("Release"); + cmd.add("--nologo"); + // No build servers left running behind a build that has ended: + // they would hold the build directory open, on Windows against + // the next clean. + cmd.add("-nodeReuse:false"); + cmd.add("-p:UseSharedCompilation=false"); + cmd.add(csproj.getAbsolutePath()); + if (exec(cmd, logFile, true) != 0) { + List out = lines(logFile); + List errors = new ArrayList(); + for (String line : out) { + if (line.contains(": error ") && !errors.contains(line.trim())) { + errors.add(line.trim()); + } + } + throw new BuildException("The C# scripts in src/main/unity/Assets did not compile:\n" + + (errors.isEmpty() ? tail(out, 40) : tail(errors, 40)) + "\n(full log: " + logFile + ")\n" + + "A script that compiles in Unity and not here uses a part of the UnityEngine API the" + + " compatibility runtime does not have."); + } + } + + public List translate(File out, File runtimeClasses, List references, List assemblies, + File logFile) throws BuildException { + List cmd = java(TRANSLATOR_MAIN); + cmd.add("--out"); + cmd.add(out.getAbsolutePath()); + cmd.add("--runtime"); + cmd.add(runtimeClasses.getAbsolutePath()); + for (File r : references) { + cmd.add("--ref"); + cmd.add(r.getAbsolutePath()); + } + for (File a : assemblies) { + cmd.add(a.getAbsolutePath()); + } + if (exec(cmd, logFile, false) != 0) { + throw new BuildException("The C# scripts in src/main/unity could not be translated:\n" + + tail(lines(logFile), 40) + "\n(full log: " + logFile + ")"); + } + return lines(logFile); + } + + public List compileScenes(File project, File sceneJavaOut, File resources, List references, + List assemblies, File logFile) throws BuildException { + List cmd = java(SCENE_COMPILER_MAIN); + cmd.add("--project"); + cmd.add(project.getAbsolutePath()); + cmd.add("--out"); + cmd.add(sceneJavaOut.getAbsolutePath()); + cmd.add("--resources"); + cmd.add(resources.getAbsolutePath()); + for (File r : references) { + cmd.add("--ref"); + cmd.add(r.getAbsolutePath()); + } + for (File a : assemblies) { + cmd.add(a.getAbsolutePath()); + } + int exit = exec(cmd, logFile, false); + List out = lines(logFile); + if (exit != 0) { + throw new BuildException("The scenes in src/main/unity could not be compiled:\n" + tail(out, 40) + + "\n(full log: " + logFile + ")"); + } + return out; + } + + private List java(String mainClassName) { + List cmd = new ArrayList(); + cmd.add(new File(new File(System.getProperty("java.home"), "bin"), isWindows() ? "java.exe" : "java") + .getAbsolutePath()); + // The scene compiler reads images with ImageIO; without this a + // desktop JVM shows an icon in the dock for the length of a build. + cmd.add("-Djava.awt.headless=true"); + cmd.add("-cp"); + StringBuilder cp = new StringBuilder(); + for (File f : toolClasspath) { + if (cp.length() > 0) { + cp.append(File.pathSeparatorChar); + } + cp.append(f.getAbsolutePath()); + } + cmd.add(cp.toString()); + cmd.add(mainClassName); + return cmd; + } + + /// Runs a command to its end, its output in `logFile`. + private int exec(List cmd, File logFile, boolean sdk) throws BuildException { + mkdirs(workDir); + ProcessBuilder pb = new ProcessBuilder(cmd); + // Not the project directory: a global.json there, written for the + // Unity editor's own tooling, would pick the SDK for us. + pb.directory(workDir); + pb.redirectErrorStream(true); + pb.redirectOutput(logFile); + if (sdk) { + Map env = pb.environment(); + env.put("DOTNET_CLI_TELEMETRY_OPTOUT", "1"); + env.put("DOTNET_NOLOGO", "1"); + env.put("DOTNET_CLI_USE_MSBUILD_SERVER", "0"); + env.put("MSBUILDDISABLENODEREUSE", "1"); + // Left alone when set: a CI machine that points these at a + // cache of its own has said where they belong. + if (!env.containsKey("DOTNET_CLI_HOME")) { + env.put("DOTNET_CLI_HOME", new File(workDir, "dotnet-home").getAbsolutePath()); + } + if (!env.containsKey("NUGET_PACKAGES")) { + env.put("NUGET_PACKAGES", new File(workDir, "nuget-packages").getAbsolutePath()); + } + } + log.debug("Running " + cmd); + try { + return pb.start().waitFor(); + } catch (IOException e) { + throw new BuildException("Cannot run " + cmd.get(0) + ": " + e.getMessage(), e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new BuildException("Interrupted while running " + cmd.get(0), e); + } + } + } + + private static List lines(File f) { + String text = read(f); + List out = new ArrayList(); + if (text != null) { + for (String line : text.split("\n")) { + out.add(line.endsWith("\r") ? line.substring(0, line.length() - 1) : line); + } + } + return out; + } + + private static String tail(List lines, int count) { + StringBuilder sb = new StringBuilder(); + for (int i = Math.max(0, lines.size() - count); i < lines.size(); i++) { + if (sb.length() > 0) { + sb.append('\n'); + } + sb.append(lines.get(i)); + } + return sb.toString(); + } +} diff --git a/maven/build-engine/src/main/java/com/codename1/maven/UnityProjectImporter.java b/maven/build-engine/src/main/java/com/codename1/maven/UnityProjectImporter.java new file mode 100644 index 00000000000..35de35ee6cc --- /dev/null +++ b/maven/build-engine/src/main/java/com/codename1/maven/UnityProjectImporter.java @@ -0,0 +1,241 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.maven; + +import com.codename1.build.Log; +import com.codename1.builders.BuildException; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; + +/// Copies a Unity project into a Codename One application +/// (`common/src/main/unity`) and makes the application's main class run it. +/// Shared by the Maven goal `import-unity-project` and the Gradle task. +/// +/// Only `Assets` and `ProjectSettings` are copied. They are the project; +/// `Library`, `Temp`, `obj` and `Logs` are what the Unity editor derives from +/// them, are routinely gigabytes, and hold the editor's own compiled +/// assemblies, which must never reach a build of ours. +/// +/// The copy is a mirror of the Unity project, not a fork of it: importing +/// again replaces every file an earlier import copied and removes the ones +/// the project no longer has. The game is edited in Unity and imported again. +public final class UnityProjectImporter { + + /// The directories of a Unity project that are its source. + static final String[] COPIED = {"Assets", "ProjectSettings"}; + + /// Names never copied, wherever they are: version-control metadata and + /// what an operating system leaves in a directory it has shown. + static final List SKIPPED = Collections.unmodifiableList(Arrays.asList( + ".git", ".svn", ".hg", ".vs", ".idea", ".DS_Store", "Thumbs.db", "obj", "Temp", "Library")); + + /// Compiled code and native libraries a project carries as files. None of + /// it is translated: there is C# source for none of it. + private static final String[] BINARY_PLUGINS = {".dll", ".so", ".a", ".dylib", ".bundle", ".aar", ".jar"}; + + /// The list of files the last import copied, kept in the target so the + /// next import can remove what the project dropped. + static final String IMPORT_RECORD = ".unity-import-files"; + + public static final class Result { + public int copiedFiles; + public int removedFiles; + public int scripts; + public int scenes; + /// `productName` of `ProjectSettings/ProjectSettings.asset`, or null. + public String productName; + /// Files that will be ignored by the build, each with the reason. + public final List ignored = new ArrayList(); + } + + private final Log log; + + public UnityProjectImporter(Log log) { + this.log = log; + } + + /// @param source the Unity project: the directory that holds `Assets` + /// @param commonDir the application's common module + /// @param mainPackage `codename1.packageName`, or null to leave the main class alone + /// @param mainClass `codename1.mainName`, or null to leave the main class alone + public Result importProject(File source, File commonDir, String mainPackage, String mainClass) + throws BuildException { + if (source == null || !source.isDirectory()) { + throw new BuildException(source + " is not a directory; name the Unity project to import, the directory" + + " that holds Assets and ProjectSettings"); + } + if (!UnityProjectBuilder.isUnityProject(source)) { + throw new BuildException(source + " is not a Unity project: it needs both an Assets and a" + + " ProjectSettings directory"); + } + File target = new File(commonDir, "src/main/unity"); + Result r = new Result(); + try { + if (target.getCanonicalFile().equals(source.getCanonicalFile())) { + throw new BuildException(source + " is already this application's src/main/unity"); + } + Set previous = readImportRecord(target); + Set imported = new TreeSet(); + for (String name : COPIED) { + copy(new File(source, name), new File(target, name), name, imported, r); + } + for (String path : previous) { + if (!imported.contains(path)) { + File stale = new File(target, path); + if (stale.isFile() && stale.delete()) { + r.removedFiles++; + } + } + } + writeImportRecord(target, imported); + r.productName = productName(new File(source, "ProjectSettings/ProjectSettings.asset")); + if (mainPackage != null && mainClass != null && mainPackage.length() > 0 && mainClass.length() > 0) { + writeEntryPoint(new File(commonDir, "src/main/java"), mainPackage, mainClass); + } + } catch (IOException e) { + throw new BuildException("Cannot import " + source + ": " + e.getMessage(), e); + } + return r; + } + + private void copy(File src, File dest, String path, Set imported, Result r) throws IOException { + if (src.isDirectory()) { + File[] children = src.listFiles(); + if (children == null) { + return; + } + for (File f : children) { + if (SKIPPED.contains(f.getName())) { + continue; + } + copy(f, new File(dest, f.getName()), path + "/" + f.getName(), imported, r); + } + return; + } + File parent = dest.getParentFile(); + if (parent != null && !parent.isDirectory() && !parent.mkdirs() && !parent.isDirectory()) { + throw new IOException("cannot create " + parent); + } + Files.copy(src.toPath(), dest.toPath(), StandardCopyOption.REPLACE_EXISTING); + imported.add(path); + r.copiedFiles++; + String name = src.getName(); + if (name.endsWith(".cs")) { + r.scripts++; + } else if (name.endsWith(".unity")) { + r.scenes++; + } else { + for (String suffix : BINARY_PLUGINS) { + if (name.endsWith(suffix)) { + r.ignored.add(path + " (compiled code: only C# source is translated)"); + break; + } + } + } + } + + /// The value of the `productName:` line of a project settings file, or + /// null. The file is YAML when the project serializes assets as text, + /// which is the only form the build reads either. + static String productName(File settings) { + if (!settings.isFile()) { + return null; + } + try { + for (String line : Files.readAllLines(settings.toPath(), StandardCharsets.ISO_8859_1)) { + String t = line.trim(); + if (t.startsWith("productName:")) { + String value = t.substring("productName:".length()).trim(); + return value.length() == 0 ? null : value; + } + } + } catch (IOException e) { + return null; + } + return null; + } + + private void writeEntryPoint(File javaDir, String pkg, String cls) throws IOException { + File f = new File(javaDir, pkg.replace('.', File.separatorChar) + File.separator + cls + ".java"); + File parent = f.getParentFile(); + if (parent != null && !parent.isDirectory() && !parent.mkdirs() && !parent.isDirectory()) { + throw new IOException("cannot create " + parent); + } + String src = UnityProjectBuilder.mainClassSource(pkg, cls, "", + "/// The Codename One entry point: runs the first scene of the imported Unity project.\n"); + byte[] bytes = src.getBytes(StandardCharsets.UTF_8); + if (f.isFile()) { + File backup = new File(f.getPath() + ".pre-unity-import"); + boolean unchanged = Arrays.equals(bytes, Files.readAllBytes(f.toPath())); + if (backup.exists()) { + // A repeated import: the backup already holds the application's + // original class, and overwriting it would lose the only copy. + // The file itself is what an earlier import wrote -- unless the + // developer has customized it since, and then it is kept. + if (!unchanged) { + log.warn(f.getName() + " was changed after an earlier import and was left as it is; " + + "delete it and import again to regenerate it"); + return; + } + } else if (!unchanged) { + Files.copy(f.toPath(), backup.toPath()); + log.info("Saved the previous " + f.getName() + " as " + backup.getName()); + } + } + Files.write(f.toPath(), bytes); + } + + private static Set readImportRecord(File target) throws IOException { + Set out = new TreeSet(); + File record = new File(target, IMPORT_RECORD); + if (record.isFile()) { + for (String line : Files.readAllLines(record.toPath(), StandardCharsets.UTF_8)) { + // Only paths under the two copied directories: the record is + // a file in the project, and a line edited into it must not + // be able to name a file elsewhere for deletion. + if (line.indexOf("..") < 0 && (line.startsWith("Assets/") || line.startsWith("ProjectSettings/"))) { + out.add(line); + } + } + } + return out; + } + + private static void writeImportRecord(File target, Set imported) throws IOException { + StringBuilder sb = new StringBuilder(); + for (String path : imported) { + sb.append(path).append('\n'); + } + Files.write(new File(target, IMPORT_RECORD).toPath(), sb.toString().getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/maven/build-engine/src/test/java/com/codename1/maven/GradleConversionTest.java b/maven/build-engine/src/test/java/com/codename1/maven/GradleConversionTest.java index e44a1cfcb1c..498f6b76f44 100644 --- a/maven/build-engine/src/test/java/com/codename1/maven/GradleConversionTest.java +++ b/maven/build-engine/src/test/java/com/codename1/maven/GradleConversionTest.java @@ -422,6 +422,31 @@ void mavenPathPatternsMatchAsMavenDoes() { assertFalse(GradleConversion.antMatches("a?c.txt", "a/c.txt")); } + /// The Gradle plugin adds the Unity runtime to a project that has + /// `src/main/unity`, at its own version and with the translator of that + /// version; the declaration cn1:import-unity-project wrote into the pom + /// must not be carried over as a second one, while the Unity project + /// itself has to arrive. + @Test + void theUnityRuntimeIsLeftToThePluginAndTheProjectIsCopied() throws Exception { + File mvn = mavenApp(UNTOUCHED_API); + touch(mvn, "common/pom.xml", "com.acmemvnapp" + + "1.0mvnapp-common" + + "com.codenameonecodenameone-unity-compat" + + "1" + + "org.exampleutil2.0" + + ""); + touch(mvn, "common/src/main/unity/Assets/Scripts/Player.cs", "class Player {}"); + touch(mvn, "common/src/main/unity/ProjectSettings/ProjectSettings.asset", "x"); + File out = new File(tmp.toFile(), "out"); + converter().convert(mvn, out, "1.0"); + String build = read(new File(out, "build.gradle.kts")); + assertFalse(build.contains("codenameone-unity-compat"), build); + assertTrue(build.contains(" implementation(\"org.example:util:2.0\")"), build); + assertTrue(new File(out, "src/main/unity/Assets/Scripts/Player.cs").isFile()); + assertTrue(new File(out, "src/main/unity/ProjectSettings/ProjectSettings.asset").isFile()); + } + @Test void platformModuleDependenciesAreNotDroppedSilently() throws Exception { File mvn = mavenApp(UNTOUCHED_API); diff --git a/maven/build-engine/src/test/java/com/codename1/maven/UnityProjectBuilderTest.java b/maven/build-engine/src/test/java/com/codename1/maven/UnityProjectBuilderTest.java new file mode 100644 index 00000000000..4d128f06a4a --- /dev/null +++ b/maven/build-engine/src/test/java/com/codename1/maven/UnityProjectBuilderTest.java @@ -0,0 +1,922 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.maven; + +import com.codename1.build.SystemStreamLog; +import com.codename1.builders.BuildException; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/// The build of `src/main/unity`, with the three external steps faked: what +/// is under test is everything around them -- when they run, what reaches the +/// classes directory, what leaves it -- and that has to be testable on a +/// machine with no .NET SDK. The steps themselves run for real in +/// `maven/integration-tests/unity-compat-project-test.sh`. +public class UnityProjectBuilderTest { + + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + private File unity; + private File target; + private File classes; + private File generated; + private File sources; + private File runtimeJar; + private File referencesJar; + private FakeToolchain tools; + private final List warned = new ArrayList(); + + /// Stands in for the SDK, the translator and the scene compiler, writing + /// the files the real ones would. + private static final class FakeToolchain implements UnityProjectBuilder.Toolchain { + int compiled; + int translated; + int scenes; + boolean failTranslation; + List classNames = new ArrayList(Arrays.asList("global/Player", "global/Shot", + "UnityEngine/Vector2")); + String csproj; + File runtimeClasses; + /// Appended to every class the fake translator writes, so that a + /// test can change a class without changing its length. + String classBody = ""; + + public void compileScripts(File csprojFile, File logFile) throws BuildException { + compiled++; + try { + csproj = new String(Files.readAllBytes(csprojFile.toPath()), StandardCharsets.UTF_8); + File dll = new File(csprojFile.getParentFile(), "bin/Release/netstandard2.1/Assembly-CSharp.dll"); + dll.getParentFile().mkdirs(); + Files.write(dll.toPath(), new byte[] {'M', 'Z'}); + } catch (IOException e) { + throw new BuildException(e.getMessage(), e); + } + } + + public List translate(File out, File runtime, List references, List assemblies, + File logFile) throws BuildException { + translated++; + runtimeClasses = runtime; + if (failTranslation) { + throw new BuildException("translation failed"); + } + try { + for (String name : classNames) { + File f = new File(out, name + ".class"); + f.getParentFile().mkdirs(); + Files.write(f.toPath(), (name + classBody).getBytes(StandardCharsets.UTF_8)); + } + } catch (IOException e) { + throw new BuildException(e.getMessage(), e); + } + return Arrays.asList("3 classes", "warning: Player.OnGUI (Player.cs:7): never called"); + } + + public List compileScenes(File project, File javaOut, File resources, List references, + List assemblies, File logFile) throws BuildException { + scenes++; + try { + File impl = new File(javaOut, "com/codename1/generated/unity/UnityAppImpl.java"); + impl.getParentFile().mkdirs(); + Files.write(impl.toPath(), "package com.codename1.generated.unity;\n".getBytes(StandardCharsets.UTF_8)); + resources.mkdirs(); + Files.write(new File(resources, "sheet.png").toPath(), new byte[] {1, 2, 3}); + } catch (IOException e) { + throw new BuildException(e.getMessage(), e); + } + return Arrays.asList("note: mouse axes read as zero", "warning: a particle system was left out", + "wrote UnityAppImpl.java"); + } + } + + @Before + public void project() throws Exception { + File root = tmp.newFolder("app"); + unity = new File(root, "src/main/unity"); + project(unity); + target = new File(root, "target"); + classes = new File(target, "classes"); + generated = new File(target, "generated-sources/unity"); + sources = new File(root, "src/main/java"); + runtimeJar = zip(new File(root, "codenameone-unity-compat.jar"), "UnityEngine/Vector2.class", + "com/codename1/unitycompat/unityengine/UnityRuntime.class", "META-INF/MANIFEST.MF"); + referencesJar = zip(new File(root, "codenameone-unity-compat-references.jar"), "netstandard.dll", + "UnityEngine.dll", "Codename1.UnityValues.dll"); + tools = new FakeToolchain(); + } + + private static void project(File unity) throws IOException { + write(new File(unity, "Assets/Scripts/Player.cs"), "class Player {}"); + write(new File(unity, "Assets/Scenes/Main.unity"), "%YAML 1.1"); + write(new File(unity, "ProjectSettings/EditorBuildSettings.asset"), "scenes"); + } + + private UnityProjectBuilder builder() { + return builder(runtimeJar, referencesJar); + } + + private UnityProjectBuilder builder(File runtime, File references) { + UnityProjectBuilder b = new UnityProjectBuilder(unity, generated, classes, target, runtime, references, + Collections.emptyList(), null, "com.acme.game", "MyGame", Collections.singletonList(sources), + new SystemStreamLog() { + @Override + public void warn(CharSequence content) { + warned.add(content.toString()); + super.warn(content); + } + }); + b.setToolchain(tools); + return b; + } + + private static void write(File f, String text) throws IOException { + f.getParentFile().mkdirs(); + Files.write(f.toPath(), text.getBytes(StandardCharsets.UTF_8)); + } + + private static File zip(File f, String... entries) throws IOException { + try (ZipOutputStream out = new ZipOutputStream(new FileOutputStream(f))) { + for (String e : entries) { + out.putNextEntry(new ZipEntry(e)); + out.write(e.getBytes(StandardCharsets.UTF_8)); + out.closeEntry(); + } + } + return f; + } + + @Test + public void aProjectWithoutUnitySourcesIsLeftAlone() throws Exception { + File empty = tmp.newFolder("plain"); + assertFalse(UnityProjectBuilder.isUnityProject(new File(empty, "src/main/unity"))); + assertFalse(UnityProjectBuilder.isUnityProject(null)); + unity = new File(empty, "src/main/unity"); + // Not even the dependency is asked for: a project that is not a Unity + // project must build exactly as it did before the goal existed. + assertFalse(builder(null, null).run()); + assertEquals(0, tools.compiled); + assertFalse(target.exists()); + } + + /// A project deleted from the sources takes its output with it. Nothing + /// else removes the translated classes and the images from the classes + /// directory -- no compiler put them there -- so without this they are + /// packaged into every target until somebody runs a clean build. + @Test + public void aDeletedProjectTakesItsOutputWithIt() throws Exception { + assertTrue(builder().run()); + // What javac makes of the generated source, and classes that are the + // application's own, in the same directory. + write(new File(classes, "com/codename1/generated/unity/UnityAppImpl.class"), "scenes"); + write(new File(classes, "com/acme/game/MyGame.class"), "main"); + write(new File(classes, "com/acme/game/Score.class"), "own"); + write(new File(classes, "global/Mine.class"), "own"); + write(new File(generated, "com/acme/other/Kept.java"), "own"); + assertTrue(new File(target, "unity/state.txt").isFile()); + + deleteTree(unity); + assertFalse(builder(null, null).run()); + + assertFalse(new File(classes, "global/Player.class").exists()); + assertFalse(new File(classes, "global/Shot.class").exists()); + assertFalse(new File(classes, "sheet.png").exists()); + assertFalse(new File(classes, "com/codename1/generated").exists()); + assertFalse(new File(classes, "com/acme/game/MyGame.class").exists()); + assertFalse(new File(generated, "com/codename1").exists()); + assertFalse(new File(generated, "com/acme/game").exists()); + assertFalse("the state file goes too, or a project that comes back is not compiled", + new File(target, "unity").exists()); + // Only what the build wrote: these are not in its record. + assertTrue(new File(classes, "com/acme/game/Score.class").isFile()); + assertTrue(new File(classes, "global/Mine.class").isFile()); + assertTrue(new File(generated, "com/acme/other/Kept.java").isFile()); + // None of it needed a tool: the first build is the only one counted. + assertEquals(1, tools.compiled); + assertEquals(1, tools.translated); + assertEquals(1, tools.scenes); + + // Nothing left to remove, and nothing made for having looked. + assertFalse(builder(null, null).removeOutput()); + assertFalse(new File(target, "unity").exists()); + + // The project put back is built from nothing. + project(unity); + assertTrue(builder().run()); + assertEquals(2, tools.compiled); + assertTrue(new File(classes, "global/Player.class").isFile()); + } + + /// Half a project is no project: `ProjectSettings` deleted alone leaves + /// `Assets` behind, and the output goes all the same. + @Test + public void aProjectLeftWithoutItsSettingsTakesItsOutputWithIt() throws Exception { + assertTrue(builder().run()); + deleteTree(new File(unity, "ProjectSettings")); + assertFalse(builder().run()); + assertFalse(new File(classes, "global/Player.class").exists()); + assertFalse(new File(generated, "com/codename1/generated/unity/UnityAppImpl.java").exists()); + assertFalse(new File(target, "unity").exists()); + assertEquals(1, tools.compiled); + } + + private static void deleteTree(File f) throws IOException { + File[] children = f.listFiles(); + if (children != null) { + for (File c : children) { + deleteTree(c); + } + } + Files.deleteIfExists(f.toPath()); + } + + /// `Assets` alone is some other directory of that name: the scenes to + /// build are listed in `ProjectSettings`. + @Test + public void assetsAloneAreNotAProject() throws Exception { + File dir = tmp.newFolder("half"); + assertTrue(new File(dir, "Assets").mkdirs()); + assertFalse(UnityProjectBuilder.isUnityProject(dir)); + unity = dir; + assertFalse(builder().run()); + assertEquals(0, tools.compiled); + } + + @Test + public void installsClassesImagesSceneSourceAndMainClass() throws Exception { + assertTrue(builder().run()); + assertTrue(new File(classes, "global/Player.class").isFile()); + assertTrue(new File(classes, "global/Shot.class").isFile()); + assertTrue(new File(classes, "sheet.png").isFile()); + assertTrue(new File(generated, "com/codename1/generated/unity/UnityAppImpl.java").isFile()); + String main = new String(Files.readAllBytes(new File(generated, "com/acme/game/MyGame.java").toPath()), + StandardCharsets.UTF_8); + assertTrue(main, main.contains("public class MyGame extends com.codename1.unitycompat.app.UnityApplication")); + assertTrue(main, main.contains("com.codename1.generated.unity.UnityAppImpl.install();")); + // The translator checks the scripts against the runtime's classes, + // which it reads from a directory: the jar was unpacked for it. + assertTrue(new File(tools.runtimeClasses, "com/codename1/unitycompat/unityengine/UnityRuntime.class").isFile()); + assertFalse("only classes are unpacked", new File(tools.runtimeClasses, "META-INF/MANIFEST.MF").exists()); + } + + /// What the translator could not carry over reaches the build's own + /// output beside what the scene compiler could not, and not only a log + /// file under `target`. + @Test + public void warningsOfBothToolsAreReported() throws Exception { + assertTrue(builder().run()); + assertEquals(Arrays.asList("src/main/unity: Player.OnGUI (Player.cs:7): never called", + "src/main/unity: a particle system was left out"), warned); + } + + /// The value types are translated with the scripts, and the runtime jar + /// already ships them: the application must not carry a second copy. + @Test + public void classesTheRuntimeShipsAreNotInstalledAgain() throws Exception { + assertTrue(builder().run()); + assertFalse(new File(classes, "UnityEngine/Vector2.class").exists()); + } + + @Test + public void anUnchangedProjectIsNotCompiledAgain() throws Exception { + assertTrue(builder().run()); + assertTrue(builder().run()); + assertEquals(1, tools.compiled); + assertEquals(1, tools.translated); + assertEquals(1, tools.scenes); + // A changed script is. + write(new File(unity, "Assets/Scripts/Player.cs"), "class Player { int lives; }"); + assertTrue(builder().run()); + assertEquals(2, tools.compiled); + // And so is a changed setting, which only the scene compiler reads. + write(new File(unity, "ProjectSettings/EditorBuildSettings.asset"), "scenes: two"); + assertTrue(builder().run()); + assertEquals(3, tools.scenes); + } + + /// An edit that keeps a file's length and its modification time -- a + /// checkout that restores times, two saves inside one tick of the file + /// system's clock -- is still an edit: text is compared by content. + @Test + public void anEditOfTheSameLengthAndTimeCompilesAgain() throws Exception { + File script = new File(unity, "Assets/Scripts/Player.cs"); + File scene = new File(unity, "Assets/Scenes/Main.unity"); + write(script, "class Player { int speed = 1; }"); + assertTrue(builder().run()); + long length = script.length(); + long time = script.lastModified(); + write(script, "class Player { int speed = 2; }"); + assertTrue(script.setLastModified(time)); + assertEquals(length, script.length()); + assertEquals(time, script.lastModified()); + assertTrue(builder().run()); + assertEquals(2, tools.compiled); + // A scene, which only the scene compiler reads, likewise. + time = scene.lastModified(); + write(scene, "%YAML 1.2"); + assertTrue(scene.setLastModified(time)); + assertTrue(builder().run()); + assertEquals(3, tools.scenes); + // And nothing changed is still nothing to do. + assertTrue(builder().run()); + assertEquals(3, tools.scenes); + } + + /// A class the translator wrote again with the same length, at a time no + /// later than the installed copy's -- one tick of the file system's + /// clock, or an installed copy whose time is ahead -- still replaces it: + /// whatever a step that ran has written is installed. + @Test + public void aRebuiltClassOfTheSameLengthAndTimeIsInstalled() throws Exception { + tools.classBody = "1"; + assertTrue(builder().run()); + File installed = new File(classes, "global/Player.class"); + assertEquals("global/Player1", new String(Files.readAllBytes(installed.toPath()), StandardCharsets.UTF_8)); + assertTrue(installed.setLastModified(System.currentTimeMillis() + 60000L)); + tools.classBody = "2"; + write(new File(unity, "Assets/Scripts/Player.cs"), "class Player { int lives; }"); + assertTrue(builder().run()); + assertEquals(2, tools.translated); + assertEquals("global/Player2", new String(Files.readAllBytes(installed.toPath()), StandardCharsets.UTF_8)); + // Nothing changed since is still nothing to copy: the installed + // class keeps a time a copy made now would not have. + File staged = new File(target, "unity/app/global/Player.class"); + assertTrue(staged.isFile()); + assertTrue(installed.setLastModified(staged.lastModified() + 5000L)); + long time = installed.lastModified(); + assertTrue(builder().run()); + assertEquals(2, tools.translated); + assertEquals(time, installed.lastModified()); + } + + /// The staging directory is made where the .NET SDK is and may be used + /// somewhere else: another checkout, at another path, with the times a + /// checkout gives its files and jars that were packed again. Nothing of + /// that is the project, so the build there has nothing to compile. + @Test + public void aCopyElsewhereWithNewTimesIsNotCompiledAgain() throws Exception { + write(new File(unity, "Assets/Sprites/ship.png"), "pixels"); + assertTrue(builder().run()); + File elsewhere = new File(tmp.newFolder("another"), "deeper/checkout"); + long later = System.currentTimeMillis() + 86400000L; + copy(unity, new File(elsewhere, "src/main/unity"), later); + copy(target, new File(elsewhere, "target"), later); + // The same entries, packed again under other names: a jar's own + // bytes differ by the times in it. + File runtime = zip(new File(elsewhere, "runtime-of-another-version.jar"), "META-INF/MANIFEST.MF", + "com/codename1/unitycompat/unityengine/UnityRuntime.class", "UnityEngine/Vector2.class"); + File references = zip(new File(elsewhere, "references.jar"), "netstandard.dll", "UnityEngine.dll", + "Codename1.UnityValues.dll"); + assertTrue(runtime.setLastModified(later)); + File classesThere = new File(elsewhere, "target/classes"); + assertTrue(new File(classesThere, "global/Player.class").delete()); + UnityProjectBuilder there = new UnityProjectBuilder(new File(elsewhere, "src/main/unity"), + new File(elsewhere, "target/generated-sources/unity"), classesThere, new File(elsewhere, "target"), + runtime, references, Collections.emptyList(), null, "com.acme.game", "MyGame", + Collections.singletonList(new File(elsewhere, "src/main/java")), new SystemStreamLog()); + there.setToolchain(tools); + assertTrue(there.run()); + assertEquals(1, tools.compiled); + assertEquals(1, tools.translated); + assertEquals(1, tools.scenes); + assertTrue(new File(classesThere, "global/Player.class").isFile()); + // And content is still what decides: a picture of the same length + // and time, with other pixels in it. + File picture = new File(elsewhere, "src/main/unity/Assets/Sprites/ship.png"); + write(picture, "PIXELS"); + assertTrue(picture.setLastModified(later)); + assertTrue(there.run()); + assertEquals(2, tools.scenes); + } + + private static void copy(File from, File to, long time) throws IOException { + File[] children = from.listFiles(); + if (children == null) { + to.getParentFile().mkdirs(); + Files.copy(from.toPath(), to.toPath()); + assertTrue(to.setLastModified(time)); + return; + } + for (File c : children) { + copy(c, new File(to, c.getName()), time); + } + } + + /// A new runtime translates differently and checks against different + /// classes: the staging directory of the old one cannot be trusted. + @Test + public void aNewRuntimeJarCompilesAgain() throws Exception { + assertTrue(builder().run()); + zip(runtimeJar, "UnityEngine/Vector2.class", "UnityEngine/Vector3.class"); + assertTrue(builder().run()); + assertEquals(2, tools.translated); + } + + /// The classes directory is the build tool's to clean; cleaning it alone + /// must not need the .NET SDK to get the game back. + /// A state file that matches is not output that is whole: each thing a + /// full build leaves under `target/unity` and install reads is required + /// before the build is skipped, or a cache restored in part installs an + /// application without its images, or with a second copy of each class + /// the runtime ships. + @Test + public void incompleteOutputIsCompiledAgain() throws Exception { + assertTrue(builder().run()); + File work = new File(target, "unity"); + String[] gone = {"resources", "runtime", "resources/sheet.png", "app/global/Player.class"}; + for (int i = 0; i < gone.length; i++) { + deleteTree(new File(work, gone[i])); + assertTrue(gone[i], builder().run()); + assertEquals(gone[i], i + 2, tools.compiled); + assertTrue(gone[i], new File(work, gone[i]).exists()); + } + // Emptied rather than deleted. + File[] runtime = new File(work, "runtime").listFiles(); + for (File f : runtime) { + deleteTree(f); + } + assertTrue(builder().run()); + assertEquals(gone.length + 2, tools.compiled); + assertFalse(new File(classes, "UnityEngine/Vector2.class").exists()); + } + + /// A project that draws no image has an empty images directory, which + /// is whole. + @Test + public void aProjectWithoutImagesIsNotCompiledAgain() throws Exception { + assertTrue(builder().run()); + File resources = new File(target, "unity/resources"); + assertTrue(new File(resources, "sheet.png").delete()); + // As a build of a project without images records its install. + Files.write(new File(target, "unity/installed.txt").toPath(), + "global/Player.class\nglobal/Shot.class\n".getBytes(StandardCharsets.UTF_8)); + assertTrue(builder().run()); + assertEquals(1, tools.compiled); + } + + @Test + public void aCleanedClassesDirectoryIsRefilledWithoutCompiling() throws Exception { + assertTrue(builder().run()); + assertTrue(new File(classes, "global/Player.class").delete()); + assertTrue(new File(classes, "sheet.png").delete()); + assertTrue(builder().run()); + assertEquals(1, tools.compiled); + assertTrue(new File(classes, "global/Player.class").isFile()); + assertTrue(new File(classes, "sheet.png").isFile()); + } + + @Test + public void aScriptDeletedFromTheProjectLeavesTheClasses() throws Exception { + assertTrue(builder().run()); + write(new File(classes, "com/acme/game/Own.class"), "the application's own class"); + tools.classNames.remove("global/Shot"); + write(new File(unity, "Assets/Scripts/Player.cs"), "class Player { /* no shots */ }"); + assertTrue(builder().run()); + assertFalse(new File(classes, "global/Shot.class").exists()); + assertTrue(new File(classes, "global/Player.class").isFile()); + assertTrue("only what this build installed is removed", new File(classes, "com/acme/game/Own.class").isFile()); + } + + /// A build that fails half way has replaced part of the staging + /// directory; the next one must not skip on the strength of the old state. + @Test + public void aFailedBuildIsNotRememberedAsDone() throws Exception { + assertTrue(builder().run()); + write(new File(unity, "Assets/Scripts/Player.cs"), "class Player { broken }"); + tools.failTranslation = true; + try { + builder().run(); + fail("the translation failure was swallowed"); + } catch (BuildException expected) { + assertEquals("translation failed", expected.getMessage()); + } + tools.failTranslation = false; + assertTrue(builder().run()); + assertEquals(3, tools.compiled); + assertTrue(new File(classes, "global/Player.class").isFile()); + } + + @Test + public void theApplicationsOwnMainClassRetiresTheGeneratedOne() throws Exception { + assertTrue(builder().run()); + File generatedMain = new File(generated, "com/acme/game/MyGame.java"); + assertTrue(generatedMain.isFile()); + File own = new File(sources, "com/acme/game/MyGame.java"); + write(own, "package com.acme.game; public class MyGame {}"); + assertTrue(builder().run()); + assertFalse("javac would see two MyGame classes", generatedMain.exists()); + // Deleted again, the generated one comes back, with nothing compiled. + assertTrue(own.delete()); + assertTrue(builder().run()); + assertTrue(generatedMain.isFile()); + assertEquals(1, tools.compiled); + } + + @Test + public void aMissingRuntimeDependencyIsNamed() throws Exception { + try { + builder(null, referencesJar).run(); + fail(); + } catch (BuildException e) { + assertTrue(e.getMessage(), e.getMessage().contains("codenameone-unity-compat dependency is missing")); + assertTrue(e.getMessage(), e.getMessage().contains("cn1:import-unity-project")); + } + try { + builder(runtimeJar, null).run(); + fail(); + } catch (BuildException e) { + assertTrue(e.getMessage(), e.getMessage().contains("'references' classifier")); + } + } + + @Test + public void aReferencesJarWithoutTheAssembliesIsRejected() throws Exception { + zip(referencesJar, "netstandard.dll"); + try { + builder().run(); + fail(); + } catch (BuildException e) { + assertTrue(e.getMessage(), e.getMessage().contains("has no UnityEngine.dll")); + } + assertEquals("nothing was compiled against half the references", 0, tools.compiled); + } + + /// The generated project references the three assemblies and nothing + /// else, so the SDK has nothing to download; and it compiles the scripts + /// where they are, without the editor's. + @Test + public void theCSharpProjectIsSelfContained() throws Exception { + assertTrue(builder().run()); + String p = tools.csproj; + assertTrue(p, p.contains("true")); + assertTrue(p, p.contains("netstandard2.1")); + assertTrue(p, p.contains("/Assets/**/*.cs\" Exclude=\"")); + assertTrue(p, p.contains("/Assets/**/Editor/**/*.cs\"")); + for (String name : new String[] {"netstandard", "UnityEngine", "Codename1.UnityValues"}) { + assertTrue(p, p.contains("")); + assertTrue(p, p.contains("/unity/ref/" + name + ".dll")); + } + assertFalse("no package may be needed", p.contains("PackageReference")); + } + + /// A path is data to MSBuild only when its special characters are + /// encoded; unencoded, `$(x)` in a directory name is a property reference. + @Test + public void pathsAreLiteralToMsbuild() { + assertEquals("/Users/a & b/50%25/%24(x)/it%27s%3B%40", UnityProjectBuilder.msbuild("/Users/a & b/50%/$(x)/it's;@")); + assertEquals("C:/games/My Game", UnityProjectBuilder.msbuild("C:\\games\\My Game")); + } + + @Test + public void findsDotnetWhereItIsNamedThenDotnetRootThenThePath() throws Exception { + String exe = UnityProjectBuilder.isWindows() ? "dotnet.exe" : "dotnet"; + File named = new File(tmp.newFolder("named"), exe); + File rooted = new File(tmp.newFolder("rooted"), exe); + File onPath = new File(tmp.newFolder("onpath"), exe); + for (File f : new File[] {named, rooted, onPath}) { + write(f, ""); + } + Map env = new HashMap(); + env.put("DOTNET_ROOT", rooted.getParent()); + env.put("PATH", tmp.getRoot() + File.pathSeparator + onPath.getParent()); + assertEquals(named, UnityProjectBuilder.locateDotnet(named.getPath(), env)); + assertEquals("the directory it is in names it too", named, + UnityProjectBuilder.locateDotnet(named.getParent(), env)); + assertEquals(rooted, UnityProjectBuilder.locateDotnet(null, env)); + assertEquals(rooted, UnityProjectBuilder.locateDotnet(" ", env)); + env.remove("DOTNET_ROOT"); + assertEquals(onPath, UnityProjectBuilder.locateDotnet(null, env)); + assertEquals("a bare command is looked for on the PATH", onPath, + UnityProjectBuilder.locateDotnet(exe, env)); + } + + @Test + public void aMissingSdkSaysWhereItLookedAndWhatToDo() { + Map env = new HashMap(); + env.put("PATH", tmp.getRoot().getPath()); + try { + UnityProjectBuilder.locateDotnet(null, env); + fail(); + } catch (BuildException e) { + String m = e.getMessage(); + assertTrue(m, m.contains(".NET SDK, which was not found")); + assertTrue(m, m.contains("DOTNET_ROOT environment variable (not set)")); + assertTrue(m, m.contains("-Dcn1.unity.dotnet=/path/to/dotnet")); + assertTrue(m, m.contains("https://dotnet.microsoft.com/download")); + } + // A setting that names nothing is an error, not a reason to fall back: + // the developer said which SDK to use. + try { + UnityProjectBuilder.locateDotnet(new File(tmp.getRoot(), "no/such/dotnet").getPath(), env); + fail(); + } catch (BuildException e) { + assertTrue(e.getMessage(), e.getMessage().startsWith("cn1.unity.dotnet names ")); + } + } + + /// A build that reads its project from a configured directory names that + /// directory, not the default one it does not have. + @Test + public void aMissingSdkNamesTheProjectThatNeedsIt() { + Map env = new HashMap(); + env.put("PATH", tmp.getRoot().getPath()); + File elsewhere = new File(tmp.getRoot(), "samples/gallery"); + try { + UnityProjectBuilder.locateDotnet(null, env, elsewhere); + fail(); + } catch (BuildException e) { + String m = e.getMessage(); + assertTrue(m, m.startsWith(elsewhere.getPath() + " holds a Unity project")); + assertFalse(m, m.contains("src/main/unity")); + } + } + + /// Output compiled on another machine is used only with the exact jars + /// and project files it was compiled from. A build that has such output + /// and still needs the SDK says that the output does not match, since + /// installing an SDK is not what its owner meant to do. + @Test + public void outputOfOtherInputsIsNamedWhenTheSdkIsMissing() throws Exception { + builder().run(); + File state = new File(target, "unity/state.txt"); + assertTrue(state.isFile()); + write(new File(unity, "Assets/Later.cs"), "class Later {}"); + File noSdk = new File(tmp.getRoot(), "no/such/dotnet"); + UnityProjectBuilder real = new UnityProjectBuilder(unity, generated, classes, target, runtimeJar, + referencesJar, Collections.singletonList(runtimeJar), noSdk.getPath(), "com.acme.game", "MyGame", + Collections.singletonList(sources), new SystemStreamLog()); + try { + real.run(); + fail(); + } catch (BuildException e) { + String m = e.getMessage(); + assertTrue(m, m.startsWith("cn1.unity.dotnet names ")); + assertTrue(m, m.contains("does not match these sources or jars")); + assertTrue(m, m.contains(state.getPath())); + assertTrue(m, m.contains(unity.getPath())); + } + // The first build of a project has no output to explain. + assertFalse(state.exists()); + try { + real.run(); + fail(); + } catch (BuildException e) { + assertFalse(e.getMessage(), e.getMessage().contains("does not match")); + } + } + + /// A builder with a tool class path, whose INFO lines are kept. + private UnityProjectBuilder told(File tool, final List said) { + UnityProjectBuilder b = new UnityProjectBuilder(unity, generated, classes, target, runtimeJar, referencesJar, + Collections.singletonList(tool), null, "com.acme.game", "MyGame", Collections.singletonList(sources), + new SystemStreamLog() { + @Override + public void info(CharSequence content) { + said.add(content.toString()); + } + }); + b.setToolchain(tools); + return b; + } + + private static String mismatch(List said) { + for (String line : said) { + if (line.contains("does not match these sources or jars")) { + return line; + } + } + return null; + } + + /// Output compiled on another machine that is refused here has to say + /// which of its four inputs is not the one it was compiled from. The + /// first time it happened the log said only that one of them was, from a + /// runner nobody could look at afterwards. + @Test + public void aMismatchNamesThePartAndTheFirstEntryThatDiffer() throws Exception { + File tool = zip(new File(tmp.getRoot(), "tool.jar"), "com/acme/Translator.class", "org/asm/Reader.class"); + List said = new ArrayList(); + assertTrue(told(tool, said).run()); + assertEquals(said.toString(), null, mismatch(said)); + // Nothing changed: nothing is said, and nothing is compiled. + assertTrue(told(tool, said).run()); + assertEquals(said.toString(), null, mismatch(said)); + assertEquals(1, tools.compiled); + + // The project: one file edited, one added. + write(new File(unity, "Assets/Scripts/Player.cs"), "class Player { int lives; }"); + write(new File(unity, "Assets/Later.cs"), "class Later {}"); + assertTrue(told(tool, said).run()); + String m = mismatch(said); + assertTrue(m, m.contains("What differs: the project files (")); + assertTrue(m, m.contains("; 2 of 4 entries differ, the first: Assets/Later.cs is there now and is not" + + " recorded).")); + assertFalse(m, m.contains("; the codenameone-")); + assertFalse(m, m.contains("; the 'references' jar")); + + // A file that is gone, and one whose content is another. + said.clear(); + assertTrue(new File(unity, "Assets/Later.cs").delete()); + assertTrue(told(tool, said).run()); + m = mismatch(said); + assertTrue(m, m.contains("1 of 4 entries differ, the first: Assets/Later.cs is recorded and is not there" + + " now).")); + said.clear(); + write(new File(unity, "Assets/Scripts/Player.cs"), "class Player { int score; }"); + assertTrue(told(tool, said).run()); + m = mismatch(said); + assertTrue(m, m.contains("1 of 3 entries differ, the first: Assets/Scripts/Player.cs has other content).")); + + // The runtime jar. + said.clear(); + zip(runtimeJar, "UnityEngine/Vector2.class", "UnityEngine/Vector3.class", + "com/codename1/unitycompat/unityengine/UnityRuntime.class", "META-INF/MANIFEST.MF"); + assertTrue(told(tool, said).run()); + m = mismatch(said); + assertTrue(m, m.contains("What differs: the codenameone-unity-compat jar (")); + assertTrue(m, m.contains("1 of 3 entries differ, the first: UnityEngine/Vector3.class is there now and is" + + " not recorded).")); + assertFalse(m, m.contains("the project files")); + + // The references. + said.clear(); + zip(referencesJar, "netstandard.dll", "UnityEngine.dll", "Codename1.UnityValues.dll", "Other.dll"); + assertTrue(told(tool, said).run()); + m = mismatch(said); + assertTrue(m, m.contains("What differs: the 'references' jar (")); + assertTrue(m, m.contains("the first: Other.dll is there now and is not recorded).")); + + // The tool, whose jars are one class path: the same classes from + // another jar of another name are the same content. + said.clear(); + File renamed = zip(new File(tmp.getRoot(), "tool-2.jar"), "com/acme/Translator.class", + "org/asm/Reader.class"); + assertTrue(told(renamed, said).run()); + assertEquals(said.toString(), null, mismatch(said)); + File other = zip(new File(tmp.getRoot(), "tool-3.jar"), "com/acme/Translator.class"); + assertTrue(told(other, said).run()); + m = mismatch(said); + assertTrue(m, m.contains("What differs: the codenameone-cil-translator class path (")); + assertTrue(m, m.contains("1 of 2 entries differ, the first: org/asm/Reader.class is recorded and is not" + + " there now).")); + + // Two parts at once are both named, in the order they are listed. + said.clear(); + write(new File(unity, "Assets/Scripts/Player.cs"), "class Player {}"); + assertTrue(told(tool, said).run()); + m = mismatch(said); + assertTrue(m, m.indexOf("What differs: the project files (") > 0); + assertTrue(m, m.indexOf("; the codenameone-cil-translator class path (") + > m.indexOf("What differs: the project files (")); + } + + /// The first line of the state file is the digest and the only line a + /// build is skipped on; what follows is read back as it was written. + @Test + public void theStateFileRecordsThePartsUnderTheDigest() throws Exception { + assertTrue(builder().run()); + File state = new File(target, "unity/state.txt"); + String text = new String(Files.readAllBytes(state.toPath()), StandardCharsets.UTF_8); + UnityProjectBuilder.State recorded = UnityProjectBuilder.State.parse(text); + assertEquals(text, text.substring(0, text.indexOf('\n')), recorded.digest); + assertEquals(64, recorded.digest.length()); + assertEquals(UnityProjectBuilder.STATE_VERSION, recorded.version); + assertEquals(Arrays.asList("project", "runtime", "references", "tool"), + new ArrayList(recorded.parts.keySet())); + // Three files of the project, two classes of the runtime (META-INF + // is not content), three assemblies, and no tool in this fixture. + assertEquals(text, 8, recorded.entries.size()); + assertEquals(text, recorded.text()); + assertEquals(recorded.digest, UnityProjectBuilder.State.of(recorded.entries).digest); + // Line endings a checkout or an archive rewrote do not make it + // another record. + UnityProjectBuilder.State crlf = UnityProjectBuilder.State.parse(text.replace("\n", "\r\n")); + assertEquals(recorded.digest, crlf.digest); + assertEquals(recorded.entries, crlf.entries); + assertEquals(recorded.parts, crlf.parts); + assertEquals(null, UnityProjectBuilder.State.parse(null)); + } + + /// A state file with nothing under its digest -- an older build's, or one + /// that was overwritten -- is refused as any other, and says that it has + /// nothing to compare. + @Test + public void aStateFileOfADigestAloneSaysItCannotNameThePart() throws Exception { + assertTrue(builder().run()); + File state = new File(target, "unity/state.txt"); + write(state, "stale\n"); + List said = new ArrayList(); + assertTrue(told(runtimeJar, said).run()); + String m = mismatch(said); + assertTrue(m, m.contains("Which of them differs cannot be said: the file records a digest (stale, now ")); + assertEquals(2, tools.compiled); + // An empty file is not the digest of anything, either. + write(state, ""); + said.clear(); + assertTrue(told(runtimeJar, said).run()); + assertTrue(said.toString(), mismatch(said).contains("records a digest (none, now ")); + assertEquals(3, tools.compiled); + } + + /// The same content under another version of the record: the digest + /// covers the version, and no part is blamed for it. + @Test + public void aRecordOfAnotherVersionWithTheSameContentBlamesNoPart() throws Exception { + UnityProjectBuilder.State now = UnityProjectBuilder.State.of(Arrays.asList( + "Assets/A.cs=00", "runtime!a/B.class:1:ff")); + UnityProjectBuilder.State old = UnityProjectBuilder.State.parse( + now.text().replace("version=" + UnityProjectBuilder.STATE_VERSION, "version=0") + .replace(now.digest, "0123")); + String said = UnityProjectBuilder.differences(old, now); + assertEquals("All four are the same content: the file was written as version 0 of this record and the" + + " build writes version " + UnityProjectBuilder.STATE_VERSION + ".", said); + // And a part the file gives a digest for, without its entries. + UnityProjectBuilder.State bare = UnityProjectBuilder.State.parse("0123\nversion=3\npart.project=aa\n" + + "part.runtime=" + now.parts.get("runtime") + "\npart.references=" + now.parts.get("references") + + "\npart.tool=" + now.parts.get("tool") + "\nruntime!a/B.class:1:ff\n"); + said = UnityProjectBuilder.differences(bare, now); + assertTrue(said, said.startsWith("What differs: the project files (aa recorded, ")); + assertTrue(said, said.endsWith("; the file lists none of its 1 entries).")); + } + + @Test + public void readsTheSdkVersion() { + assertEquals(10, UnityProjectBuilder.sdkMajor(Arrays.asList("10.0.401"))); + assertEquals(8, UnityProjectBuilder.sdkMajor(Arrays.asList("", "Welcome to .NET!", " 8.0.100-rc.1 "))); + assertEquals(-1, UnityProjectBuilder.sdkMajor(Arrays.asList("No .NET SDKs were found.", "Download: x"))); + assertEquals(-1, UnityProjectBuilder.sdkMajor(Collections.emptyList())); + } + + /// The real toolchain against a stand-in `dotnet`: an SDK too old for the + /// scripts, and a runtime with no SDK at all, are both refused before + /// anything is compiled, each with what to install. + @Test + public void refusesAnSdkThatIsTooOldOrIsNoSdk() throws Exception { + Assume.assumeFalse("the stand-in is a shell script", UnityProjectBuilder.isWindows()); + File old = script("old", "echo 3.1.426"); + File runtimeOnly = script("runtime", "echo 'No .NET SDKs were found.'; exit 145"); + assertRefused(old, "is the .NET SDK 3.1.426, which is too old"); + assertRefused(runtimeOnly, "A .NET runtime alone cannot compile C#"); + } + + private File script(String dir, String body) throws IOException { + File f = new File(tmp.newFolder(dir), "dotnet"); + write(f, "#!/bin/sh\n" + body + "\n"); + assertTrue(f.setExecutable(true)); + return f; + } + + private void assertRefused(File dotnet, String expected) { + UnityProjectBuilder b = new UnityProjectBuilder(unity, generated, classes, target, runtimeJar, referencesJar, + Collections.singletonList(runtimeJar), dotnet.getPath(), null, null, null, new SystemStreamLog()); + try { + b.run(); + fail("built with " + dotnet); + } catch (BuildException e) { + assertTrue(e.getMessage(), e.getMessage().contains(expected)); + assertTrue(e.getMessage(), e.getMessage().contains("Install the .NET SDK")); + } + } +} diff --git a/maven/build-engine/src/test/java/com/codename1/maven/UnityProjectImporterTest.java b/maven/build-engine/src/test/java/com/codename1/maven/UnityProjectImporterTest.java new file mode 100644 index 00000000000..553f72608df --- /dev/null +++ b/maven/build-engine/src/test/java/com/codename1/maven/UnityProjectImporterTest.java @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.maven; + +import com.codename1.build.SystemStreamLog; +import com.codename1.builders.BuildException; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class UnityProjectImporterTest { + + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + private File game; + private File common; + private File unity; + + @Before + public void project() throws Exception { + game = tmp.newFolder("MyGame"); + write(new File(game, "Assets/Scripts/Player.cs"), "class Player {}"); + write(new File(game, "Assets/Scripts/Player.cs.meta"), "guid: 1"); + write(new File(game, "Assets/Scenes/Main.unity"), "%YAML 1.1"); + write(new File(game, "Assets/Plugins/Fast.dll"), "MZ"); + write(new File(game, "ProjectSettings/ProjectSettings.asset"), + "PlayerSettings:\n companyName: Acme\n productName: Rock Fall\n"); + // What the editor derives, and what version control keeps. + write(new File(game, "Library/ScriptAssemblies/Assembly-CSharp.dll"), "MZ"); + write(new File(game, "Temp/lock"), ""); + write(new File(game, "obj/Debug/x.cache"), ""); + write(new File(game, ".git/HEAD"), "ref"); + write(new File(game, "Assets/.git/HEAD"), "a nested checkout"); + write(new File(game, "Assets/Scripts/obj/Debug/y.cache"), ""); + write(new File(game, "Packages/manifest.json"), "{}"); + common = tmp.newFolder("app", "common"); + unity = new File(common, "src/main/unity"); + } + + private static void write(File f, String text) throws IOException { + f.getParentFile().mkdirs(); + Files.write(f.toPath(), text.getBytes(StandardCharsets.UTF_8)); + } + + private static String read(File f) throws IOException { + return new String(Files.readAllBytes(f.toPath()), StandardCharsets.UTF_8); + } + + private UnityProjectImporter.Result importIt() throws BuildException { + return new UnityProjectImporter(new SystemStreamLog()).importProject(game, common, "com.acme.game", "MyGame"); + } + + @Test + public void copiesTheProjectAndNothingTheEditorDerives() throws Exception { + UnityProjectImporter.Result r = importIt(); + assertTrue(new File(unity, "Assets/Scripts/Player.cs").isFile()); + assertTrue("the .meta files carry the guids scenes refer to scripts by", + new File(unity, "Assets/Scripts/Player.cs.meta").isFile()); + assertTrue(new File(unity, "Assets/Scenes/Main.unity").isFile()); + assertTrue(new File(unity, "ProjectSettings/ProjectSettings.asset").isFile()); + for (String never : new String[] {"Library", "Temp", "obj", ".git", "Packages", "Assets/.git", + "Assets/Scripts/obj"}) { + assertFalse(never + " was copied", new File(unity, never).exists()); + } + assertEquals(5, r.copiedFiles); + assertEquals(1, r.scripts); + assertEquals(1, r.scenes); + assertEquals("Rock Fall", r.productName); + assertEquals(1, r.ignored.size()); + assertTrue(r.ignored.get(0), r.ignored.get(0).startsWith("Assets/Plugins/Fast.dll (compiled code")); + assertTrue(UnityProjectBuilder.isUnityProject(unity)); + } + + @Test + public void writesTheEntryPointAndKeepsTheApplicationsOwn() throws Exception { + File main = new File(common, "src/main/java/com/acme/game/MyGame.java"); + write(main, "package com.acme.game; public class MyGame { /* the archetype's */ }"); + importIt(); + String src = read(main); + assertTrue(src, src.startsWith("package com.acme.game;\n\n/// The Codename One entry point")); + assertTrue(src, src.contains("public class MyGame extends com.codename1.unitycompat.app.UnityApplication {")); + assertTrue(src, src.contains("com.codename1.generated.unity.UnityAppImpl.install();")); + File backup = new File(main.getPath() + ".pre-unity-import"); + assertTrue(read(backup).contains("the archetype's")); + // Importing twice keeps the backup of the application's own class; + // the second run must not replace it with what the first one wrote. + importIt(); + assertTrue(read(backup).contains("the archetype's")); + // A class customized after the import is kept as it is. + write(main, src + "// mine\n"); + importIt(); + assertTrue(read(main).endsWith("// mine\n")); + } + + /// The copy mirrors the Unity project: what the project dropped goes, + /// what changed is taken, and what the import never copied stays. + @Test + public void importingAgainMirrorsTheProject() throws Exception { + importIt(); + File own = new File(unity, "Assets/Scripts/Added.cs"); + write(own, "class Added {}"); + assertTrue(new File(game, "Assets/Scenes/Main.unity").delete()); + write(new File(game, "Assets/Scripts/Player.cs"), "class Player { int lives; }"); + UnityProjectImporter.Result r = importIt(); + assertEquals(1, r.removedFiles); + assertFalse(new File(unity, "Assets/Scenes/Main.unity").exists()); + assertTrue(read(new File(unity, "Assets/Scripts/Player.cs")).contains("lives")); + assertTrue("a file the import did not copy is not its to remove", own.isFile()); + } + + /// The record of what was imported is a file in the project; a line + /// edited into it must not name a file elsewhere for deletion. + @Test + public void theImportRecordCannotNameFilesOutsideTheCopy() throws Exception { + importIt(); + File outside = new File(common, "src/main/java/Keep.java"); + write(outside, "class Keep {}"); + File record = new File(unity, UnityProjectImporter.IMPORT_RECORD); + write(record, read(record) + "../java/Keep.java\nAssets/../../java/Keep.java\n"); + importIt(); + assertTrue(outside.isFile()); + } + + @Test + public void refusesWhatIsNotAUnityProject() throws Exception { + File notOne = tmp.newFolder("notes"); + try { + new UnityProjectImporter(new SystemStreamLog()).importProject(notOne, common, "a", "B"); + fail(); + } catch (BuildException e) { + assertTrue(e.getMessage(), e.getMessage().contains("is not a Unity project")); + } + assertFalse("nothing is created for a project that was not imported", unity.exists()); + } + + @Test + public void readsNoProductNameFromABinarySettingsFile() throws Exception { + File settings = new File(game, "ProjectSettings/ProjectSettings.asset"); + Files.write(settings.toPath(), new byte[] {0, 1, 2, (byte) 0xff, 3}); + assertNull(UnityProjectImporter.productName(settings)); + assertNull(UnityProjectImporter.productName(new File(game, "ProjectSettings/None.asset"))); + } +} diff --git a/maven/cil-translator/pom.xml b/maven/cil-translator/pom.xml new file mode 100644 index 00000000000..58566b0f431 --- /dev/null +++ b/maven/cil-translator/pom.xml @@ -0,0 +1,206 @@ + + + + + com.codenameone + codenameone + 8.0-SNAPSHOT + + 4.0.0 + + codenameone-cil-translator + 8.0-SNAPSHOT + jar + codenameone-cil-translator + + Build-time translator from .NET CIL to JVM bytecode. Reads the + assemblies a C# compiler produced (ECMA-335 metadata and method + bodies) and writes class files every Codename One target already + knows how to build: the simulator and Android run them as they are, + ParparVM translates them on to C and JavaScript. Depends on nothing + but ASM. + + + + UTF-8 + 1.8 + 1.8 + + + + + + maven-compiler-plugin + + 1.8 + 1.8 + + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.6.1 + + + copy-translator-dependencies + process-classes + + copy-dependencies + + + runtime + ${project.build.directory}/dependency + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + + ${project.basedir}/../.. + ${project.build.directory}/unity-tests + + + + + + com.github.spotbugs + spotbugs-maven-plugin + 4.7.3.4 + + Max + Low + true + true + ${project.build.directory} + ${project.basedir}/spotbugs-exclude.xml + + + + spotbugs + verify + + spotbugs + check + + + + + + org.apache.maven.plugins + maven-pmd-plugin + 3.21.0 + + false + false + 1.8 + + ${project.basedir}/../core-unittests/pmd.xml + + + + + pmd + verify + + pmd + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 3.3.0 + + ${project.basedir}/../core-unittests/checkstyle.xml + ${project.build.sourceEncoding} + false + false + false + false + ${project.build.directory}/checkstyle-result.xml + xml + + + + validate + verify + + checkstyle + + + + + checkstyle-zero-errors + verify + + check + + + true + error + 0 + true + + + + + + + + + + org.ow2.asm + asm + + + + com.codenameone + codenameone-core + test + + + junit + junit + test + + + + org.junit.vintage + junit-vintage-engine + ${junit.jupiter.version} + test + + + diff --git a/maven/cil-translator/spotbugs-exclude.xml b/maven/cil-translator/spotbugs-exclude.xml new file mode 100644 index 00000000000..718bf0b7eaf --- /dev/null +++ b/maven/cil-translator/spotbugs-exclude.xml @@ -0,0 +1,64 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/maven/cil-translator/src/main/java/com/codename1/cil/metadata/CilAssembly.java b/maven/cil-translator/src/main/java/com/codename1/cil/metadata/CilAssembly.java new file mode 100644 index 00000000000..24fa9f2f304 --- /dev/null +++ b/maven/cil-translator/src/main/java/com/codename1/cil/metadata/CilAssembly.java @@ -0,0 +1,775 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.cil.metadata; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/// The logical layer of one assembly: its types, their members, and the +/// answer to "what does this token mean". +/// +/// An assembly is read once, eagerly, apart from method bodies. References to +/// types in other assemblies are resolved by name through the [Universe] the +/// assembly was loaded into, which is the only way to learn whether a +/// referenced type is a class, a struct or an interface: a reference row +/// records the name and nothing else. +public final class CilAssembly { + + public static final int TYPE_INTERFACE = 0x20; + public static final int TYPE_ABSTRACT = 0x80; + public static final int TYPE_SEALED = 0x100; + + public static final int METHOD_ACCESS_MASK = 0x7; + public static final int METHOD_PRIVATE = 0x1; + public static final int METHOD_STATIC = 0x10; + public static final int METHOD_FINAL = 0x20; + public static final int METHOD_VIRTUAL = 0x40; + public static final int METHOD_NEW_SLOT = 0x100; + public static final int METHOD_ABSTRACT = 0x400; + + public static final int FIELD_STATIC = 0x10; + public static final int FIELD_LITERAL = 0x40; + + public static final int SEMANTICS_ADD_ON = 0x8; + public static final int SEMANTICS_REMOVE_ON = 0x10; + + /// A type this assembly defines. + public static final class TypeDef { + public final CilAssembly assembly; + public final int row; + public final String namespace; + public final String name; + public final int flags; + public TypeDef enclosing; + /// Null only for `System.Object` and for interfaces. + public CilType baseType; + public final List interfaces = new ArrayList(); + public final List fields = new ArrayList(); + public final List methods = new ArrayList(); + public final List methodImpls = new ArrayList(); + public int genericParamCount; + private String fullName; + + TypeDef(CilAssembly assembly, int row, String namespace, String name, int flags) { + this.assembly = assembly; + this.row = row; + this.namespace = namespace; + this.name = name; + this.flags = flags; + } + + public String fullName() { + if (fullName == null) { + if (enclosing != null) { + fullName = enclosing.fullName() + "/" + name; + } else { + fullName = namespace.length() == 0 ? name : namespace + "." + name; + } + } + return fullName; + } + + public boolean isInterface() { + return (flags & TYPE_INTERFACE) != 0; + } + + public boolean isAbstract() { + return (flags & TYPE_ABSTRACT) != 0; + } + + private String baseName() { + return baseType == null ? null : baseType.typeName(); + } + + public boolean isEnum() { + return "System.Enum".equals(baseName()); + } + + /// A struct or an enum. `System.Enum` itself derives from + /// `System.ValueType` and is a class. + public boolean isValueType() { + String base = baseName(); + if ("System.Enum".equals(base)) { + return true; + } + return "System.ValueType".equals(base) && !"System.Enum".equals(fullName()); + } + + public boolean isDelegate() { + return "System.MulticastDelegate".equals(baseName()); + } + + /// The integer type an enum is stored as: the type of its one + /// instance field. + public CilType enumUnderlyingType() { + for (FieldDef f : fields) { + if (!f.isStatic()) { + return f.type; + } + } + throw new CilFormatException(assembly.source() + ": enum " + fullName() + " has no value field"); + } + + public CilType asType() { + CilType builtin = CilType.builtin(fullName()); + return builtin != null ? builtin : CilType.named(fullName(), isValueType()); + } + + public FieldDef field(String fieldName) { + for (FieldDef f : fields) { + if (f.name.equals(fieldName)) { + return f; + } + } + return null; + } + + @Override + public String toString() { + return fullName(); + } + } + + public static final class FieldDef { + public final TypeDef owner; + public final int row; + public final String name; + public final int flags; + public final CilType type; + /// Where the initial data of a mapped field lives, or 0. + public int rva; + /// The value of a literal field of an integer type -- a member of an + /// enum, a `const int` -- or null. + public Long constant; + + FieldDef(TypeDef owner, int row, String name, int flags, CilType type) { + this.owner = owner; + this.row = row; + this.name = name; + this.flags = flags; + this.type = type; + } + + public boolean isStatic() { + return (flags & FIELD_STATIC) != 0; + } + + public boolean isLiteral() { + return (flags & FIELD_LITERAL) != 0; + } + + public int token() { + return (MetadataReader.FIELD << 24) | row; + } + } + + public static final class MethodDef { + public final TypeDef owner; + public final int row; + public final String name; + public final int flags; + public final int rva; + public final MethodSig sig; + public int genericParamCount; + /// `SEMANTICS_*` bits when this is an event accessor, else 0. + public int semantics; + /// Name of the event or property this method is an accessor of. + public String associationName; + private MethodBody body; + + MethodDef(TypeDef owner, int row, String name, int flags, int rva, MethodSig sig) { + this.owner = owner; + this.row = row; + this.name = name; + this.flags = flags; + this.rva = rva; + this.sig = sig; + } + + public boolean isStatic() { + return (flags & METHOD_STATIC) != 0; + } + + public boolean isVirtual() { + return (flags & METHOD_VIRTUAL) != 0; + } + + public boolean isAbstract() { + return (flags & METHOD_ABSTRACT) != 0; + } + + public boolean isNewSlot() { + return (flags & METHOD_NEW_SLOT) != 0; + } + + public boolean isPrivate() { + return (flags & METHOD_ACCESS_MASK) == METHOD_PRIVATE; + } + + public boolean hasBody() { + return rva != 0; + } + + public MethodBody body() { + if (body == null) { + body = new MethodBody(owner.assembly, rva); + } + return body; + } + + public int token() { + return (MetadataReader.METHOD_DEF << 24) | row; + } + + @Override + public String toString() { + return owner.fullName() + "::" + name + sig; + } + } + + /// An explicit override: `body` is this type's implementation of + /// `declaration`, whatever the two are called. + public static final class MethodImpl { + public final MethodRef body; + public final MethodRef declaration; + + MethodImpl(MethodRef body, MethodRef declaration) { + this.body = body; + this.declaration = declaration; + } + } + + public static final class MethodSig { + public final boolean hasThis; + public final int genericParamCount; + public final CilType returnType; + public final CilType[] params; + + MethodSig(boolean hasThis, int genericParamCount, CilType returnType, CilType[] params) { + this.hasThis = hasThis; + this.genericParamCount = genericParamCount; + this.returnType = returnType; + this.params = params; + } + + public boolean sameShape(MethodSig other) { + if (hasThis != other.hasThis || genericParamCount != other.genericParamCount + || params.length != other.params.length || !returnType.equals(other.returnType)) { + return false; + } + for (int i = 0; i < params.length; i++) { + if (!params[i].equals(other.params[i])) { + return false; + } + } + return true; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("("); + for (int i = 0; i < params.length; i++) { + if (i > 0) { + sb.append(", "); + } + sb.append(params[i]); + } + return sb.append(") : ").append(returnType).toString(); + } + } + + /// A method as an instruction names it: where it is declared, what it is + /// called and its signature exactly as declared, type variables included. + /// The instantiation, when there is one, is carried beside the signature + /// and never substituted into it, because the erased JVM descriptor is + /// derived from the declaration while boxing decisions need the arguments. + public static final class MethodRef { + public final CilType declaringType; + public final String name; + public final MethodSig sig; + public final CilType[] methodArgs; + /// The definition, when the reference is to this assembly's own method. + public final MethodDef def; + + MethodRef(CilType declaringType, String name, MethodSig sig, CilType[] methodArgs, MethodDef def) { + this.declaringType = declaringType; + this.name = name; + this.sig = sig; + this.methodArgs = methodArgs; + this.def = def; + } + + public CilType[] typeArgs() { + return declaringType.kind == CilType.Kind.GENERICINST ? declaringType.args : null; + } + + @Override + public String toString() { + return declaringType + "::" + name + sig; + } + } + + public static final class FieldRef { + public final CilType declaringType; + public final String name; + public final CilType type; + public final FieldDef def; + + FieldRef(CilType declaringType, String name, CilType type, FieldDef def) { + this.declaringType = declaringType; + this.name = name; + this.type = type; + this.def = def; + } + + @Override + public String toString() { + return declaringType + "::" + name; + } + } + + private final MetadataReader md; + private final String name; + private Universe universe; + private final List types = new ArrayList(); + private final Map typesByName = new HashMap(); + private FieldDef[] fieldsByRow; + private MethodDef[] methodsByRow; + private final Map tokenCache = new HashMap(); + private Map> attributes; + private PortablePdb debug; + + public CilAssembly(byte[] data, String source) { + md = new MetadataReader(data, source); + if (md.rowCount(MetadataReader.ASSEMBLY) > 0) { + name = md.string(md.cell(MetadataReader.ASSEMBLY, 1, 7)); + } else { + name = md.string(md.cell(MetadataReader.MODULE, 1, 1)); + } + } + + public MetadataReader metadata() { + return md; + } + + /// Gives the assembly the portable PDB the compiler wrote beside it, + /// so that [#sourceLocation(MethodDef, int)] has something to answer + /// with. + public void debugInformation(PortablePdb pdb) { + debug = pdb; + } + + /// Where in the C# source an instruction of a method came from, as + /// `File.cs:line`, or null when the assembly came without a PDB or the + /// PDB does not say. + public String sourceLocation(MethodDef method, int ilOffset) { + return debug == null ? null : debug.location(method.row, ilOffset); + } + + /// The C# file a type was written in, as the path the compiler was + /// given with `/` between its parts, or null when the assembly came + /// without a PDB or no method of the type has source of its own. Read + /// from the first method that says, then from the types nested in it, + /// which is where a lambda or an iterator of the type lives. + public String sourceFile(TypeDef type) { + if (debug == null) { + return null; + } + for (MethodDef m : type.methods) { + String document = debug.document(m.row); + if (document != null) { + return document; + } + } + for (TypeDef t : types) { + if (t.enclosing == type) { // NOPMD CompareObjectsWithEquals + String document = sourceFile(t); + if (document != null) { + return document; + } + } + } + return null; + } + + public String source() { + return md.source(); + } + + public String name() { + return name; + } + + public List types() { + return Collections.unmodifiableList(types); + } + + public TypeDef type(String fullName) { + return typesByName.get(fullName); + } + + /// Reads the types. Split from construction because signatures mention + /// types of other assemblies, which have to be registered first. + void load(Universe owner) { + universe = owner; + int typeCount = md.rowCount(MetadataReader.TYPE_DEF); + for (int row = 1; row <= typeCount; row++) { + types.add(new TypeDef(this, row, md.string(md.cell(MetadataReader.TYPE_DEF, row, 2)), + md.string(md.cell(MetadataReader.TYPE_DEF, row, 1)), md.cell(MetadataReader.TYPE_DEF, row, 0))); + } + for (int row = 1; row <= md.rowCount(MetadataReader.NESTED_CLASS); row++) { + typeAt(md.cell(MetadataReader.NESTED_CLASS, row, 0)).enclosing = + typeAt(md.cell(MetadataReader.NESTED_CLASS, row, 1)); + } + for (TypeDef t : types) { + typesByName.put(t.fullName(), t); + } + } + + /// Reads what every type derives from. Whether a type is a value type is + /// read off its base, and other assemblies ask that question while they + /// link, so the bases of all assemblies are in place before any links. + void linkBases() { + for (TypeDef t : types) { + int extendsValue = md.cell(MetadataReader.TYPE_DEF, t.row, 3); + int extendsToken = md.codedToken(MetadataReader.TYPE_DEF_OR_REF, extendsValue); + if (MetadataReader.tokenRow(extendsToken) != 0) { + t.baseType = baseTypeFromToken(extendsToken); + } + } + } + + /// Reads everything that needs other assemblies' type names to be known. + void link() { + for (int row = 1; row <= md.rowCount(MetadataReader.GENERIC_PARAM); row++) { + int owner = md.codedToken(MetadataReader.TYPE_OR_METHOD_DEF, md.cell(MetadataReader.GENERIC_PARAM, row, 2)); + if (MetadataReader.tokenTable(owner) == MetadataReader.TYPE_DEF) { + typeAt(MetadataReader.tokenRow(owner)).genericParamCount++; + } + } + fieldsByRow = new FieldDef[md.rowCount(MetadataReader.FIELD) + 1]; + methodsByRow = new MethodDef[md.rowCount(MetadataReader.METHOD_DEF) + 1]; + for (TypeDef t : types) { + int fieldEnd = md.listEnd(MetadataReader.TYPE_DEF, t.row, 4, MetadataReader.FIELD); + for (int f = md.cell(MetadataReader.TYPE_DEF, t.row, 4); f < fieldEnd; f++) { + int[] blob = md.blob(md.cell(MetadataReader.FIELD, f, 2)); + SignatureParser parser = new SignatureParser(this, blob[0]); + FieldDef field = new FieldDef(t, f, md.string(md.cell(MetadataReader.FIELD, f, 1)), + md.cell(MetadataReader.FIELD, f, 0), parser.fieldSig()); + t.fields.add(field); + fieldsByRow[f] = field; + } + int methodEnd = md.listEnd(MetadataReader.TYPE_DEF, t.row, 5, MetadataReader.METHOD_DEF); + for (int m = md.cell(MetadataReader.TYPE_DEF, t.row, 5); m < methodEnd; m++) { + int[] blob = md.blob(md.cell(MetadataReader.METHOD_DEF, m, 4)); + MethodDef method = new MethodDef(t, m, md.string(md.cell(MetadataReader.METHOD_DEF, m, 3)), + md.cell(MetadataReader.METHOD_DEF, m, 2), + md.cell(MetadataReader.METHOD_DEF, m, 0), new SignatureParser(this, blob[0]).methodSig()); + t.methods.add(method); + methodsByRow[m] = method; + } + } + for (int row = 1; row <= md.rowCount(MetadataReader.GENERIC_PARAM); row++) { + int owner = md.codedToken(MetadataReader.TYPE_OR_METHOD_DEF, md.cell(MetadataReader.GENERIC_PARAM, row, 2)); + if (MetadataReader.tokenTable(owner) == MetadataReader.METHOD_DEF) { + methodsByRow[MetadataReader.tokenRow(owner)].genericParamCount++; + } + } + for (int row = 1; row <= md.rowCount(MetadataReader.INTERFACE_IMPL); row++) { + int token = md.codedToken(MetadataReader.TYPE_DEF_OR_REF, md.cell(MetadataReader.INTERFACE_IMPL, row, 1)); + typeAt(md.cell(MetadataReader.INTERFACE_IMPL, row, 0)).interfaces.add(typeFromToken(token)); + } + for (int row = 1; row <= md.rowCount(MetadataReader.FIELD_RVA); row++) { + fieldsByRow[md.cell(MetadataReader.FIELD_RVA, row, 1)].rva = md.cell(MetadataReader.FIELD_RVA, row, 0); + } + for (int row = 1; row <= md.rowCount(MetadataReader.CONSTANT); row++) { + int parent = md.codedToken(MetadataReader.HAS_CONSTANT, md.cell(MetadataReader.CONSTANT, row, 1)); + if (MetadataReader.tokenTable(parent) != MetadataReader.FIELD) { + continue; + } + int[] span = md.blob(md.cell(MetadataReader.CONSTANT, row, 2)); + int at = span[0]; + long value; + // The element type of the constant: the integers, signed and not. + switch (md.cell(MetadataReader.CONSTANT, row, 0) & 0xFF) { + case 0x02: + case 0x05: + value = md.u1(at); + break; + case 0x04: + value = (byte) md.u1(at); + break; + case 0x03: + case 0x07: + value = md.u2(at); + break; + case 0x06: + value = (short) md.u2(at); + break; + case 0x08: + value = md.i4(at); + break; + case 0x09: + value = md.i4(at) & 0xFFFFFFFFL; + break; + case 0x0A: + case 0x0B: + value = md.i8(at); + break; + default: + continue; + } + fieldsByRow[MetadataReader.tokenRow(parent)].constant = Long.valueOf(value); + } + for (int row = 1; row <= md.rowCount(MetadataReader.METHOD_SEMANTICS); row++) { + MethodDef method = methodsByRow[md.cell(MetadataReader.METHOD_SEMANTICS, row, 1)]; + int association = md.codedToken(MetadataReader.HAS_SEMANTICS, + md.cell(MetadataReader.METHOD_SEMANTICS, row, 2)); + method.semantics = md.cell(MetadataReader.METHOD_SEMANTICS, row, 0); + int table = MetadataReader.tokenTable(association); + method.associationName = md.string(md.cell(table, MetadataReader.tokenRow(association), 1)); + } + } + + /// Method overrides name methods of other types, whose own members must + /// have been read already, so this runs after every assembly is linked. + void linkOverrides() { + for (int row = 1; row <= md.rowCount(MetadataReader.METHOD_IMPL); row++) { + TypeDef owner = typeAt(md.cell(MetadataReader.METHOD_IMPL, row, 0)); + MethodRef body = methodRef(md.codedToken(MetadataReader.METHOD_DEF_OR_REF, + md.cell(MetadataReader.METHOD_IMPL, row, 1))); + MethodRef declaration = methodRef(md.codedToken(MetadataReader.METHOD_DEF_OR_REF, + md.cell(MetadataReader.METHOD_IMPL, row, 2))); + owner.methodImpls.add(new MethodImpl(body, declaration)); + } + } + + private TypeDef typeAt(int row) { + return types.get(row - 1); + } + + // The base type decides whether a type is a value type, so resolving it + // must not ask that question of the type being linked. A base is always a + // class, which is all the answer that is needed here. + private CilType baseTypeFromToken(int token) { + int table = MetadataReader.tokenTable(token); + if (table == MetadataReader.TYPE_SPEC) { + return typeFromToken(token); + } + String fullName = table == MetadataReader.TYPE_DEF ? typeAt(MetadataReader.tokenRow(token)).fullName() + : typeRefName(MetadataReader.tokenRow(token)); + CilType builtin = CilType.builtin(fullName); + return builtin != null ? builtin : CilType.named(fullName, false); + } + + String typeRefName(int row) { + int scope = md.codedToken(MetadataReader.RESOLUTION_SCOPE, md.cell(MetadataReader.TYPE_REF, row, 0)); + String typeName = md.string(md.cell(MetadataReader.TYPE_REF, row, 1)); + String namespace = md.string(md.cell(MetadataReader.TYPE_REF, row, 2)); + if (MetadataReader.tokenTable(scope) == MetadataReader.TYPE_REF && MetadataReader.tokenRow(scope) != 0) { + return typeRefName(MetadataReader.tokenRow(scope)) + "/" + typeName; + } + return namespace.length() == 0 ? typeName : namespace + "." + typeName; + } + + /// Full name of the type a TypeDef or TypeRef token names. + String typeNameFromToken(int token) { + int table = MetadataReader.tokenTable(token); + if (table == MetadataReader.TYPE_DEF) { + return typeAt(MetadataReader.tokenRow(token)).fullName(); + } + if (table == MetadataReader.TYPE_REF) { + return typeRefName(MetadataReader.tokenRow(token)); + } + throw new CilFormatException(source() + ": token 0x" + Integer.toHexString(token) + " does not name a type"); + } + + /// The type a TypeDef, TypeRef or TypeSpec token names. + public CilType typeFromToken(int token) { + Integer key = Integer.valueOf(token); + Object cached = tokenCache.get(key); + if (cached != null) { + return (CilType) cached; + } + CilType result; + int table = MetadataReader.tokenTable(token); + if (table == MetadataReader.TYPE_SPEC) { + int[] blob = md.blob(md.cell(MetadataReader.TYPE_SPEC, MetadataReader.tokenRow(token), 0)); + result = new SignatureParser(this, blob[0]).type(); + } else { + String fullName = typeNameFromToken(token); + result = CilType.builtin(fullName); + if (result == null) { + result = CilType.named(fullName, universe.require(fullName, source()).isValueType()); + } + } + tokenCache.put(key, result); + return result; + } + + CilType[] localsFromToken(int token) { + int[] blob = md.blob(md.cell(MetadataReader.STANDALONE_SIG, MetadataReader.tokenRow(token), 0)); + return new SignatureParser(this, blob[0]).locals(); + } + + /// The method a MethodDef, MemberRef or MethodSpec token names. + public MethodRef methodRef(int token) { + Integer key = Integer.valueOf(token); + Object cached = tokenCache.get(key); + if (cached != null) { + return (MethodRef) cached; + } + MethodRef result; + int table = MetadataReader.tokenTable(token); + int row = MetadataReader.tokenRow(token); + if (table == MetadataReader.METHOD_DEF) { + MethodDef def = methodsByRow[row]; + result = new MethodRef(def.owner.asType(), def.name, def.sig, null, def); + } else if (table == MetadataReader.MEMBER_REF) { + CilType parent = memberRefParent(row); + int[] blob = md.blob(md.cell(MetadataReader.MEMBER_REF, row, 2)); + result = new MethodRef(parent, md.string(md.cell(MetadataReader.MEMBER_REF, row, 1)), + new SignatureParser(this, blob[0]).methodSig(), null, null); + } else if (table == MetadataReader.METHOD_SPEC) { + MethodRef generic = methodRef(md.codedToken(MetadataReader.METHOD_DEF_OR_REF, + md.cell(MetadataReader.METHOD_SPEC, row, 0))); + int[] blob = md.blob(md.cell(MetadataReader.METHOD_SPEC, row, 1)); + result = new MethodRef(generic.declaringType, generic.name, generic.sig, + new SignatureParser(this, blob[0]).methodSpec(), generic.def); + } else { + throw new CilFormatException(source() + ": token 0x" + Integer.toHexString(token) + + " does not name a method"); + } + tokenCache.put(key, result); + return result; + } + + /// The field a Field or MemberRef token names. + public FieldRef fieldRef(int token) { + Integer key = Integer.valueOf(token); + Object cached = tokenCache.get(key); + if (cached != null) { + return (FieldRef) cached; + } + FieldRef result; + int table = MetadataReader.tokenTable(token); + int row = MetadataReader.tokenRow(token); + if (table == MetadataReader.FIELD) { + FieldDef def = fieldsByRow[row]; + result = new FieldRef(def.owner.asType(), def.name, def.type, def); + } else if (table == MetadataReader.MEMBER_REF) { + CilType parent = memberRefParent(row); + int[] blob = md.blob(md.cell(MetadataReader.MEMBER_REF, row, 2)); + result = new FieldRef(parent, md.string(md.cell(MetadataReader.MEMBER_REF, row, 1)), + new SignatureParser(this, blob[0]).fieldSig(), null); + } else { + throw new CilFormatException(source() + ": token 0x" + Integer.toHexString(token) + + " does not name a field"); + } + tokenCache.put(key, result); + return result; + } + + private CilType memberRefParent(int row) { + int parent = md.codedToken(MetadataReader.MEMBER_REF_PARENT, md.cell(MetadataReader.MEMBER_REF, row, 0)); + int table = MetadataReader.tokenTable(parent); + if (table == MetadataReader.TYPE_DEF || table == MetadataReader.TYPE_REF + || table == MetadataReader.TYPE_SPEC) { + return typeFromToken(parent); + } + throw new CilFormatException(source() + ": member reference " + row + + " has a parent this reader does not model (table 0x" + Integer.toHexString(table) + ")"); + } + + /// True when a `ldtoken` operand names a field (a static array + /// initialiser) and not a type or a method. + public boolean isFieldToken(int token) { + int table = MetadataReader.tokenTable(token); + if (table == MetadataReader.FIELD) { + return true; + } + if (table != MetadataReader.MEMBER_REF) { + return false; + } + int[] blob = md.blob(md.cell(MetadataReader.MEMBER_REF, MetadataReader.tokenRow(token), 2)); + return md.u1(blob[0]) == 0x06; + } + + /// The initial bytes of a field that has them (a compiler-generated + /// array initialiser). + public byte[] fieldData(FieldDef field, int length) { + if (field.rva == 0) { + throw new CilFormatException(source() + ": field " + field.name + " has no initial data"); + } + return md.bytes(md.rvaToOffset(field.rva), length); + } + + /// Full names of the attribute types applied to a type, field or method. + public List attributes(int token) { + if (attributes == null) { + attributes = new HashMap>(); + for (int row = 1; row <= md.rowCount(MetadataReader.CUSTOM_ATTRIBUTE); row++) { + int parent = md.codedToken(MetadataReader.HAS_CUSTOM_ATTRIBUTE, + md.cell(MetadataReader.CUSTOM_ATTRIBUTE, row, 0)); + int ctor = md.codedToken(MetadataReader.CUSTOM_ATTRIBUTE_TYPE, + md.cell(MetadataReader.CUSTOM_ATTRIBUTE, row, 1)); + String attributeType; + if (MetadataReader.tokenTable(ctor) == MetadataReader.METHOD_DEF) { + attributeType = methodsByRow[MetadataReader.tokenRow(ctor)].owner.fullName(); + } else { + int owner = md.codedToken(MetadataReader.MEMBER_REF_PARENT, + md.cell(MetadataReader.MEMBER_REF, MetadataReader.tokenRow(ctor), 0)); + int ownerTable = MetadataReader.tokenTable(owner); + if (ownerTable != MetadataReader.TYPE_DEF && ownerTable != MetadataReader.TYPE_REF) { + continue; + } + attributeType = typeNameFromToken(owner); + } + Integer key = Integer.valueOf(parent); + List list = attributes.get(key); + if (list == null) { + list = new ArrayList(); + attributes.put(key, list); + } + list.add(attributeType); + } + } + List found = attributes.get(Integer.valueOf(token)); + return found == null ? Collections.emptyList() : found; + } + + public MethodDef entryPoint() { + int token = md.entryPointToken(); + if (MetadataReader.tokenTable(token) != MetadataReader.METHOD_DEF || MetadataReader.tokenRow(token) == 0) { + return null; + } + return methodsByRow[MetadataReader.tokenRow(token)]; + } + + public String userString(int token) { + return md.userString(token); + } +} diff --git a/maven/cil-translator/src/main/java/com/codename1/cil/metadata/CilFormatException.java b/maven/cil-translator/src/main/java/com/codename1/cil/metadata/CilFormatException.java new file mode 100644 index 00000000000..67bae65b640 --- /dev/null +++ b/maven/cil-translator/src/main/java/com/codename1/cil/metadata/CilFormatException.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.cil.metadata; + +/// An assembly that cannot be read, or that uses something this reader does +/// not model. The message always names the file. +public final class CilFormatException extends RuntimeException { + private static final long serialVersionUID = 1L; + + public CilFormatException(String message) { + super(message); + } +} diff --git a/maven/cil-translator/src/main/java/com/codename1/cil/metadata/CilType.java b/maven/cil-translator/src/main/java/com/codename1/cil/metadata/CilType.java new file mode 100644 index 00000000000..c79b0b7e7c7 --- /dev/null +++ b/maven/cil-translator/src/main/java/com/codename1/cil/metadata/CilType.java @@ -0,0 +1,334 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.cil.metadata; + +import java.util.Arrays; + +/// A type as a signature spells it. Immutable, and compared structurally, so +/// two signatures read from different assemblies match when they mean the +/// same thing. +/// +/// A named type carries its full CIL name: namespace, a dot, the name, and +/// `/` between a nested type and the type enclosing it. Generic arity stays in +/// the name as the compiler wrote it (``List`1``). +public final class CilType { + public enum Kind { + VOID, BOOLEAN, CHAR, I1, U1, I2, U2, I4, U4, I8, U8, R4, R8, I, U, STRING, OBJECT, + CLASS, VALUETYPE, VAR, MVAR, SZARRAY, ARRAY, GENERICINST, BYREF, PTR, TYPEDBYREF, FNPTR + } + + // Above the constants on purpose: their constructor reads it, and a + // static declared below them is still null while they are built. + private static final CilType[] NO_ARGS = new CilType[0]; + + public static final CilType VOID = new CilType(Kind.VOID, null, null, null, 0); + public static final CilType BOOLEAN = new CilType(Kind.BOOLEAN, null, null, null, 0); + public static final CilType CHAR = new CilType(Kind.CHAR, null, null, null, 0); + public static final CilType I1 = new CilType(Kind.I1, null, null, null, 0); + public static final CilType U1 = new CilType(Kind.U1, null, null, null, 0); + public static final CilType I2 = new CilType(Kind.I2, null, null, null, 0); + public static final CilType U2 = new CilType(Kind.U2, null, null, null, 0); + public static final CilType I4 = new CilType(Kind.I4, null, null, null, 0); + public static final CilType U4 = new CilType(Kind.U4, null, null, null, 0); + public static final CilType I8 = new CilType(Kind.I8, null, null, null, 0); + public static final CilType U8 = new CilType(Kind.U8, null, null, null, 0); + public static final CilType R4 = new CilType(Kind.R4, null, null, null, 0); + public static final CilType R8 = new CilType(Kind.R8, null, null, null, 0); + public static final CilType I = new CilType(Kind.I, null, null, null, 0); + public static final CilType U = new CilType(Kind.U, null, null, null, 0); + public static final CilType STRING = new CilType(Kind.STRING, null, null, null, 0); + public static final CilType OBJECT = new CilType(Kind.OBJECT, null, null, null, 0); + public static final CilType TYPEDBYREF = new CilType(Kind.TYPEDBYREF, null, null, null, 0); + public static final CilType FNPTR = new CilType(Kind.FNPTR, null, null, null, 0); + + public final Kind kind; + /// Full name, for CLASS and VALUETYPE. + public final String name; + /// Element of an array, byref or pointer; the generic definition of a GENERICINST. + public final CilType element; + /// Type arguments of a GENERICINST. + public final CilType[] args; + /// Position of a VAR or MVAR; rank of an ARRAY. + public final int index; + + private CilType(Kind kind, String name, CilType element, CilType[] args, int index) { + this.kind = kind; + this.name = name; + this.element = element; + this.args = args == null ? NO_ARGS : args; + this.index = index; + } + + public static CilType named(String name, boolean valueType) { + return new CilType(valueType ? Kind.VALUETYPE : Kind.CLASS, name, null, null, 0); + } + + public static CilType var(int index) { + return new CilType(Kind.VAR, null, null, null, index); + } + + public static CilType mvar(int index) { + return new CilType(Kind.MVAR, null, null, null, index); + } + + public static CilType szArray(CilType element) { + return new CilType(Kind.SZARRAY, null, element, null, 1); + } + + public static CilType array(CilType element, int rank) { + return new CilType(Kind.ARRAY, null, element, null, rank); + } + + public static CilType byRef(CilType element) { + return new CilType(Kind.BYREF, null, element, null, 0); + } + + public static CilType pointer(CilType element) { + return new CilType(Kind.PTR, null, element, null, 0); + } + + public static CilType genericInst(CilType definition, CilType[] args) { + return new CilType(Kind.GENERICINST, null, definition, args, 0); + } + + /// The named type behind this one: itself for a class or value type, the + /// definition for an instantiation, null for everything else. + public CilType definition() { + if (kind == Kind.GENERICINST) { + return element; + } + return kind == Kind.CLASS || kind == Kind.VALUETYPE ? this : null; + } + + /// Full name of the named type behind this one, or the `System` name a + /// built-in stands for. Null for arrays, variables and pointers. + public String typeName() { + switch (kind) { + case CLASS: + case VALUETYPE: + return name; + case GENERICINST: + return element.name; + case BOOLEAN: + return "System.Boolean"; + case CHAR: + return "System.Char"; + case I1: + return "System.SByte"; + case U1: + return "System.Byte"; + case I2: + return "System.Int16"; + case U2: + return "System.UInt16"; + case I4: + return "System.Int32"; + case U4: + return "System.UInt32"; + case I8: + return "System.Int64"; + case U8: + return "System.UInt64"; + case R4: + return "System.Single"; + case R8: + return "System.Double"; + case I: + return "System.IntPtr"; + case U: + return "System.UIntPtr"; + case STRING: + return "System.String"; + case OBJECT: + return "System.Object"; + case VOID: + return "System.Void"; + default: + return null; + } + } + + /// The built-in a `System` name stands for, so that a reference to + /// `System.Int32` and the `int32` element type are one type. Null when + /// the name is not a built-in. + public static CilType builtin(String fullName) { + if (!fullName.startsWith("System.")) { + return null; + } + switch (fullName) { + case "System.Boolean": + return BOOLEAN; + case "System.Char": + return CHAR; + case "System.SByte": + return I1; + case "System.Byte": + return U1; + case "System.Int16": + return I2; + case "System.UInt16": + return U2; + case "System.Int32": + return I4; + case "System.UInt32": + return U4; + case "System.Int64": + return I8; + case "System.UInt64": + return U8; + case "System.Single": + return R4; + case "System.Double": + return R8; + case "System.IntPtr": + return I; + case "System.UIntPtr": + return U; + case "System.String": + return STRING; + case "System.Object": + return OBJECT; + case "System.Void": + return VOID; + default: + return null; + } + } + + public boolean isPrimitive() { + switch (kind) { + case BOOLEAN: + case CHAR: + case I1: + case U1: + case I2: + case U2: + case I4: + case U4: + case I8: + case U8: + case R4: + case R8: + case I: + case U: + return true; + default: + return false; + } + } + + /// Replaces type variables with the arguments of an instantiation. Either + /// array may be null when that level is not instantiated. + public CilType substitute(CilType[] typeArgs, CilType[] methodArgs) { + switch (kind) { + case VAR: + return typeArgs != null && index < typeArgs.length ? typeArgs[index] : this; + case MVAR: + return methodArgs != null && index < methodArgs.length ? methodArgs[index] : this; + case SZARRAY: + return szArray(element.substitute(typeArgs, methodArgs)); + case ARRAY: + return array(element.substitute(typeArgs, methodArgs), index); + case BYREF: + return byRef(element.substitute(typeArgs, methodArgs)); + case PTR: + return pointer(element.substitute(typeArgs, methodArgs)); + case GENERICINST: { + CilType[] out = new CilType[args.length]; + for (int i = 0; i < out.length; i++) { + out[i] = args[i].substitute(typeArgs, methodArgs); + } + return genericInst(element, out); + } + default: + return this; + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof CilType)) { + return false; + } + CilType t = (CilType) o; + if (kind != t.kind || index != t.index) { + return false; + } + if (name == null ? t.name != null : !name.equals(t.name)) { + return false; + } + if (element == null ? t.element != null : !element.equals(t.element)) { + return false; + } + return Arrays.equals(args, t.args); + } + + @Override + public int hashCode() { + int h = kind.ordinal() * 31 + index; + if (name != null) { + h = h * 31 + name.hashCode(); + } + if (element != null) { + h = h * 31 + element.hashCode(); + } + return h * 31 + Arrays.hashCode(args); + } + + @Override + public String toString() { + switch (kind) { + case CLASS: + return name; + case VALUETYPE: + return "valuetype " + name; + case VAR: + return "!" + index; + case MVAR: + return "!!" + index; + case SZARRAY: + return element + "[]"; + case ARRAY: + return element + "[rank " + index + "]"; + case BYREF: + return element + "&"; + case PTR: + return element + "*"; + case GENERICINST: { + StringBuilder sb = new StringBuilder(element.toString()).append('<'); + for (int i = 0; i < args.length; i++) { + if (i > 0) { + sb.append(','); + } + sb.append(args[i]); + } + return sb.append('>').toString(); + } + default: + return kind.name().toLowerCase(java.util.Locale.ROOT); + } + } +} diff --git a/maven/cil-translator/src/main/java/com/codename1/cil/metadata/MetadataReader.java b/maven/cil-translator/src/main/java/com/codename1/cil/metadata/MetadataReader.java new file mode 100644 index 00000000000..3adfea4b29d --- /dev/null +++ b/maven/cil-translator/src/main/java/com/codename1/cil/metadata/MetadataReader.java @@ -0,0 +1,467 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.cil.metadata; + +import java.nio.charset.StandardCharsets; + +/// The physical layer of a .NET assembly: the PE container, the metadata +/// streams and the metadata tables, as ECMA-335 partition II describes them. +/// +/// Nothing here interprets a row. A table is addressed by its number, a row by +/// its 1-based index and a column by its position, and the answer is the raw +/// integer stored there. [CilAssembly] turns those into types and methods. +/// +/// The schema below has to list every table the standard defines, used or not: +/// rows are packed back to back with no per-table offset, so the position of a +/// table is only known once the width of every table before it is. +public final class MetadataReader { + public static final int MODULE = 0x00; + public static final int TYPE_REF = 0x01; + public static final int TYPE_DEF = 0x02; + public static final int FIELD = 0x04; + public static final int METHOD_DEF = 0x06; + public static final int PARAM = 0x08; + public static final int INTERFACE_IMPL = 0x09; + public static final int MEMBER_REF = 0x0A; + public static final int CONSTANT = 0x0B; + public static final int CUSTOM_ATTRIBUTE = 0x0C; + public static final int CLASS_LAYOUT = 0x0F; + public static final int STANDALONE_SIG = 0x11; + public static final int EVENT_MAP = 0x12; + public static final int EVENT = 0x14; + public static final int PROPERTY_MAP = 0x15; + public static final int PROPERTY = 0x17; + public static final int METHOD_SEMANTICS = 0x18; + public static final int METHOD_IMPL = 0x19; + public static final int MODULE_REF = 0x1A; + public static final int TYPE_SPEC = 0x1B; + public static final int FIELD_RVA = 0x1D; + public static final int ASSEMBLY = 0x20; + public static final int ASSEMBLY_REF = 0x23; + public static final int FILE = 0x26; + public static final int EXPORTED_TYPE = 0x27; + public static final int NESTED_CLASS = 0x29; + public static final int GENERIC_PARAM = 0x2A; + public static final int METHOD_SPEC = 0x2B; + public static final int GENERIC_PARAM_CONSTRAINT = 0x2C; + + private static final int TABLE_COUNT = 0x2D; + + // Column kinds. A positive value is a fixed width in bytes; the heap + // indexes are negative; a simple table index is T + table and a coded + // index is C + its position in CODED. + private static final int STR = -1; + private static final int GUID = -2; + private static final int BLOB = -3; + private static final int T = 100; + private static final int C = 200; + + public static final int TYPE_DEF_OR_REF = 0; + public static final int HAS_CONSTANT = 1; + public static final int HAS_CUSTOM_ATTRIBUTE = 2; + public static final int HAS_FIELD_MARSHAL = 3; + public static final int HAS_DECL_SECURITY = 4; + public static final int MEMBER_REF_PARENT = 5; + public static final int HAS_SEMANTICS = 6; + public static final int METHOD_DEF_OR_REF = 7; + public static final int MEMBER_FORWARDED = 8; + public static final int IMPLEMENTATION = 9; + public static final int CUSTOM_ATTRIBUTE_TYPE = 10; + public static final int RESOLUTION_SCOPE = 11; + public static final int TYPE_OR_METHOD_DEF = 12; + + // The tables each coded index can point into, in tag order. -1 is a tag + // value the standard leaves unused. + private static final int[][] CODED = { + {TYPE_DEF, TYPE_REF, TYPE_SPEC}, + {FIELD, PARAM, PROPERTY}, + {METHOD_DEF, FIELD, TYPE_REF, TYPE_DEF, PARAM, INTERFACE_IMPL, MEMBER_REF, MODULE, 0x0E, PROPERTY, EVENT, + STANDALONE_SIG, MODULE_REF, TYPE_SPEC, ASSEMBLY, ASSEMBLY_REF, FILE, EXPORTED_TYPE, 0x28, GENERIC_PARAM, + GENERIC_PARAM_CONSTRAINT, METHOD_SPEC}, + {FIELD, PARAM}, + {TYPE_DEF, METHOD_DEF, ASSEMBLY}, + {TYPE_DEF, TYPE_REF, MODULE_REF, METHOD_DEF, TYPE_SPEC}, + {EVENT, PROPERTY}, + {METHOD_DEF, MEMBER_REF}, + {FIELD, METHOD_DEF}, + {FILE, ASSEMBLY_REF, EXPORTED_TYPE}, + {-1, -1, METHOD_DEF, MEMBER_REF, -1}, + {MODULE, MODULE_REF, ASSEMBLY_REF, TYPE_REF}, + {TYPE_DEF, METHOD_DEF}, + }; + + private static final int[][] COLUMNS = new int[TABLE_COUNT][]; + + static { + COLUMNS[0x00] = new int[] {2, STR, GUID, GUID, GUID}; + COLUMNS[0x01] = new int[] {C + RESOLUTION_SCOPE, STR, STR}; + COLUMNS[0x02] = new int[] {4, STR, STR, C + TYPE_DEF_OR_REF, T + FIELD, T + METHOD_DEF}; + COLUMNS[0x03] = new int[] {T + FIELD}; + COLUMNS[0x04] = new int[] {2, STR, BLOB}; + COLUMNS[0x05] = new int[] {T + METHOD_DEF}; + COLUMNS[0x06] = new int[] {4, 2, 2, STR, BLOB, T + PARAM}; + COLUMNS[0x07] = new int[] {T + PARAM}; + COLUMNS[0x08] = new int[] {2, 2, STR}; + COLUMNS[0x09] = new int[] {T + TYPE_DEF, C + TYPE_DEF_OR_REF}; + COLUMNS[0x0A] = new int[] {C + MEMBER_REF_PARENT, STR, BLOB}; + COLUMNS[0x0B] = new int[] {2, C + HAS_CONSTANT, BLOB}; + COLUMNS[0x0C] = new int[] {C + HAS_CUSTOM_ATTRIBUTE, C + CUSTOM_ATTRIBUTE_TYPE, BLOB}; + COLUMNS[0x0D] = new int[] {C + HAS_FIELD_MARSHAL, BLOB}; + COLUMNS[0x0E] = new int[] {2, C + HAS_DECL_SECURITY, BLOB}; + COLUMNS[0x0F] = new int[] {2, 4, T + TYPE_DEF}; + COLUMNS[0x10] = new int[] {4, T + FIELD}; + COLUMNS[0x11] = new int[] {BLOB}; + COLUMNS[0x12] = new int[] {T + TYPE_DEF, T + EVENT}; + COLUMNS[0x13] = new int[] {T + EVENT}; + COLUMNS[0x14] = new int[] {2, STR, C + TYPE_DEF_OR_REF}; + COLUMNS[0x15] = new int[] {T + TYPE_DEF, T + PROPERTY}; + COLUMNS[0x16] = new int[] {T + PROPERTY}; + COLUMNS[0x17] = new int[] {2, STR, BLOB}; + COLUMNS[0x18] = new int[] {2, T + METHOD_DEF, C + HAS_SEMANTICS}; + COLUMNS[0x19] = new int[] {T + TYPE_DEF, C + METHOD_DEF_OR_REF, C + METHOD_DEF_OR_REF}; + COLUMNS[0x1A] = new int[] {STR}; + COLUMNS[0x1B] = new int[] {BLOB}; + COLUMNS[0x1C] = new int[] {2, C + MEMBER_FORWARDED, STR, T + MODULE_REF}; + COLUMNS[0x1D] = new int[] {4, T + FIELD}; + COLUMNS[0x1E] = new int[] {4, 4}; + COLUMNS[0x1F] = new int[] {4}; + COLUMNS[0x20] = new int[] {4, 2, 2, 2, 2, 4, BLOB, STR, STR}; + COLUMNS[0x21] = new int[] {4}; + COLUMNS[0x22] = new int[] {4, 4, 4}; + COLUMNS[0x23] = new int[] {2, 2, 2, 2, 4, BLOB, STR, STR, BLOB}; + COLUMNS[0x24] = new int[] {4, T + ASSEMBLY_REF}; + COLUMNS[0x25] = new int[] {4, 4, 4, T + ASSEMBLY_REF}; + COLUMNS[0x26] = new int[] {4, STR, BLOB}; + COLUMNS[0x27] = new int[] {4, 4, STR, STR, C + IMPLEMENTATION}; + COLUMNS[0x28] = new int[] {4, 4, STR, C + IMPLEMENTATION}; + COLUMNS[0x29] = new int[] {T + TYPE_DEF, T + TYPE_DEF}; + COLUMNS[0x2A] = new int[] {2, 2, C + TYPE_OR_METHOD_DEF, STR}; + COLUMNS[0x2B] = new int[] {C + METHOD_DEF_OR_REF, BLOB}; + COLUMNS[0x2C] = new int[] {T + GENERIC_PARAM, C + TYPE_DEF_OR_REF}; + } + + private final byte[] data; + private final String source; + + private int[] sectionRva; + private int[] sectionSize; + private int[] sectionOffset; + + private int stringsOffset; + private int userStringsOffset; + private int blobOffset; + private boolean wideStrings; + private boolean wideGuids; + private boolean wideBlobs; + + private final int[] rowCounts = new int[TABLE_COUNT]; + private final int[] tableOffsets = new int[TABLE_COUNT]; + private final int[] rowSizes = new int[TABLE_COUNT]; + private final int[][] columnOffsets = new int[TABLE_COUNT][]; + private final int[][] columnSizes = new int[TABLE_COUNT][]; + + private int entryPointToken; + + public MetadataReader(byte[] data, String source) { + this.data = data; + this.source = source; + readPe(); + } + + public String source() { + return source; + } + + private CilFormatException bad(String what) { + return new CilFormatException(source + ": " + what); + } + + public int u1(int at) { + return data[at] & 0xFF; + } + + public int u2(int at) { + return (data[at] & 0xFF) | ((data[at + 1] & 0xFF) << 8); + } + + public int i4(int at) { + return (data[at] & 0xFF) | ((data[at + 1] & 0xFF) << 8) | ((data[at + 2] & 0xFF) << 16) + | ((data[at + 3] & 0xFF) << 24); + } + + public long i8(int at) { + return (i4(at) & 0xFFFFFFFFL) | ((long) i4(at + 4) << 32); + } + + public byte[] bytes(int at, int length) { + byte[] out = new byte[length]; + System.arraycopy(data, at, out, 0, length); + return out; + } + + private void readPe() { + if (data.length < 0x40 || u2(0) != 0x5A4D) { + throw bad("not a PE file"); + } + int pe = i4(0x3C); + if (pe <= 0 || pe + 24 > data.length || i4(pe) != 0x00004550) { + throw bad("missing PE signature"); + } + int coff = pe + 4; + int sections = u2(coff + 2); + int optionalSize = u2(coff + 16); + int optional = coff + 20; + int magic = u2(optional); + int directories; + if (magic == 0x10B) { + directories = optional + 96; + } else if (magic == 0x20B) { + directories = optional + 112; + } else { + throw bad("unknown optional header magic 0x" + Integer.toHexString(magic)); + } + int sectionTable = optional + optionalSize; + sectionRva = new int[sections]; + sectionSize = new int[sections]; + sectionOffset = new int[sections]; + for (int i = 0; i < sections; i++) { + int s = sectionTable + i * 40; + int virtualSize = i4(s + 8); + int rawSize = i4(s + 16); + sectionRva[i] = i4(s + 12); + sectionSize[i] = Math.max(virtualSize, rawSize); + sectionOffset[i] = i4(s + 20); + } + // Directory 14 is the CLI header; a native image has none. + int cliRva = i4(directories + 14 * 8); + if (cliRva == 0) { + throw bad("not a managed assembly (no CLI header)"); + } + int cli = rvaToOffset(cliRva); + int metadata = rvaToOffset(i4(cli + 8)); + entryPointToken = i4(cli + 20); + readMetadataRoot(metadata); + } + + /// Maps a relative virtual address, which is how the metadata names a + /// method body or a field's initial data, to a position in the file. + public int rvaToOffset(int rva) { + for (int i = 0; i < sectionRva.length; i++) { + if (rva >= sectionRva[i] && rva < sectionRva[i] + sectionSize[i]) { + return rva - sectionRva[i] + sectionOffset[i]; + } + } + throw bad("RVA 0x" + Integer.toHexString(rva) + " is outside every section"); + } + + private void readMetadataRoot(int root) { + if (i4(root) != 0x424A5342) { + throw bad("bad metadata signature"); + } + int versionLength = i4(root + 12); + int at = root + 16 + versionLength; + int streams = u2(at + 2); + at += 4; + int tables = -1; + for (int i = 0; i < streams; i++) { + int offset = i4(at); + at += 8; + int nameStart = at; + while (data[at] != 0) { + at++; + } + String name = new String(data, nameStart, at - nameStart, StandardCharsets.US_ASCII); + // The name is NUL terminated and then padded to a 4 byte boundary. + at = nameStart + ((at - nameStart + 4) & ~3); + if ("#~".equals(name) || "#-".equals(name)) { + tables = root + offset; + } else if ("#Strings".equals(name)) { + stringsOffset = root + offset; + } else if ("#US".equals(name)) { + userStringsOffset = root + offset; + } else if ("#Blob".equals(name)) { + blobOffset = root + offset; + } + } + if (tables < 0) { + throw bad("no metadata table stream"); + } + readTables(tables); + } + + private void readTables(int at) { + int heapSizes = u1(at + 6); + wideStrings = (heapSizes & 1) != 0; + wideGuids = (heapSizes & 2) != 0; + wideBlobs = (heapSizes & 4) != 0; + long valid = i8(at + 8); + int p = at + 24; + for (int t = 0; t < 64; t++) { + if ((valid & (1L << t)) == 0) { + continue; + } + if (t >= TABLE_COUNT) { + throw bad("unknown metadata table 0x" + Integer.toHexString(t)); + } + rowCounts[t] = i4(p); + p += 4; + } + for (int t = 0; t < TABLE_COUNT; t++) { + int[] kinds = COLUMNS[t]; + int[] offsets = new int[kinds.length]; + int[] sizes = new int[kinds.length]; + int width = 0; + for (int c = 0; c < kinds.length; c++) { + offsets[c] = width; + sizes[c] = columnSize(kinds[c]); + width += sizes[c]; + } + columnOffsets[t] = offsets; + columnSizes[t] = sizes; + rowSizes[t] = width; + tableOffsets[t] = p; + p += width * rowCounts[t]; + } + } + + private int columnSize(int kind) { + if (kind > 0 && kind < T) { + return kind; + } + if (kind == STR) { + return wideStrings ? 4 : 2; + } + if (kind == GUID) { + return wideGuids ? 4 : 2; + } + if (kind == BLOB) { + return wideBlobs ? 4 : 2; + } + if (kind >= C) { + int[] targets = CODED[kind - C]; + int max = 0; + for (int target : targets) { + if (target >= 0) { + max = Math.max(max, rowCounts[target]); + } + } + return max < (1 << (16 - tagBits(targets.length))) ? 2 : 4; + } + return rowCounts[kind - T] < 65536 ? 2 : 4; + } + + private static int tagBits(int choices) { + int bits = 0; + while ((1 << bits) < choices) { + bits++; + } + return bits; + } + + public int rowCount(int table) { + return rowCounts[table]; + } + + /// The raw value of one cell. `row` is 1-based, as every reference in the + /// metadata is; row 0 means "none" and is never stored. + public int cell(int table, int row, int column) { + if (row < 1 || row > rowCounts[table]) { + throw bad("row " + row + " is outside table 0x" + Integer.toHexString(table) + " (" + rowCounts[table] + + " rows)"); + } + int at = tableOffsets[table] + (row - 1) * rowSizes[table] + columnOffsets[table][column]; + return columnSizes[table][column] == 2 ? u2(at) : i4(at); + } + + /// Decodes a coded index into a token: the table number in the top byte + /// and the row below it, which is also how IL operands name a row. + public int codedToken(int coded, int value) { + int[] targets = CODED[coded]; + int bits = tagBits(targets.length); + int tag = value & ((1 << bits) - 1); + int row = value >>> bits; + if (tag >= targets.length || targets[tag] < 0) { + throw bad("coded index tag " + tag + " is not defined for coded index " + coded); + } + return (targets[tag] << 24) | row; + } + + public static int tokenTable(int token) { + return token >>> 24; + } + + public static int tokenRow(int token) { + return token & 0xFFFFFF; + } + + /// The first row past a row's member list. A list column (the fields or + /// methods of a type) holds only where the run starts; it ends where the + /// next row's run starts, or at the end of the member table. + public int listEnd(int table, int row, int column, int memberTable) { + if (row < rowCounts[table]) { + return cell(table, row + 1, column); + } + return rowCounts[memberTable] + 1; + } + + public String string(int index) { + int start = stringsOffset + index; + int end = start; + while (data[end] != 0) { + end++; + } + return new String(data, start, end - start, StandardCharsets.UTF_8); + } + + /// Position and length of a blob, as `{offset, length}`. + public int[] blob(int index) { + return compressedLengthPrefixed(blobOffset + index); + } + + private int[] compressedLengthPrefixed(int at) { + int first = u1(at); + if ((first & 0x80) == 0) { + return new int[] {at + 1, first}; + } + if ((first & 0xC0) == 0x80) { + return new int[] {at + 2, ((first & 0x3F) << 8) | u1(at + 1)}; + } + return new int[] {at + 4, ((first & 0x1F) << 24) | (u1(at + 1) << 16) | (u1(at + 2) << 8) | u1(at + 3)}; + } + + /// The string an `ldstr` token names. User strings are UTF-16LE with one + /// trailing flag byte that is not part of the text. + public String userString(int token) { + int[] span = compressedLengthPrefixed(userStringsOffset + tokenRow(token)); + int chars = span[1] / 2; + char[] out = new char[chars]; + for (int i = 0; i < chars; i++) { + out[i] = (char) u2(span[0] + i * 2); + } + return new String(out); + } + + public int entryPointToken() { + return entryPointToken; + } +} diff --git a/maven/cil-translator/src/main/java/com/codename1/cil/metadata/MethodBody.java b/maven/cil-translator/src/main/java/com/codename1/cil/metadata/MethodBody.java new file mode 100644 index 00000000000..24380fb7d55 --- /dev/null +++ b/maven/cil-translator/src/main/java/com/codename1/cil/metadata/MethodBody.java @@ -0,0 +1,310 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.cil.metadata; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/// The decoded body of one method: its locals, its instructions and its +/// exception clauses. +/// +/// Instructions are normalised while they are read so that a consumer sees one +/// spelling of each operation. The compact forms a compiler picks to save +/// bytes -- `ldarg.0`, `ldc.i4.s`, `br.s` -- come out as the general form with +/// the operand filled in, and a branch operand is the absolute offset of its +/// target, never the relative distance the file stores. +public final class MethodBody { + + /// One instruction. Which operand field is meaningful follows from the + /// operation: a token, an index, a 32-bit constant or a branch target is + /// in [#operand]. + public static final class Instr { + public final int offset; + /// Offset of the instruction that follows this one in the stream. + public final int next; + public final String op; + public final int operand; + public final long longOperand; + public final double doubleOperand; + public final int[] targets; + + Instr(int offset, int next, String op, int operand, long longOperand, double doubleOperand, int[] targets) { + this.offset = offset; + this.next = next; + this.op = op; + this.operand = operand; + this.longOperand = longOperand; + this.doubleOperand = doubleOperand; + this.targets = targets; + } + + @Override + public String toString() { + return "IL_" + Integer.toHexString(offset) + ": " + op; + } + } + + /// One row of the exception table. The ranges are half open: + /// `[tryStart, tryEnd)` and `[handlerStart, handlerEnd)`. + public static final class Clause { + public static final int CATCH = 0; + public static final int FILTER = 1; + public static final int FINALLY = 2; + public static final int FAULT = 4; + + public final int flags; + public final int tryStart; + public final int tryEnd; + public final int handlerStart; + public final int handlerEnd; + /// Token of the caught type for a catch clause; filter offset for a filter. + public final int classToken; + + Clause(int flags, int tryStart, int tryLength, int handlerStart, int handlerLength, int classToken) { + this.flags = flags; + this.tryStart = tryStart; + this.tryEnd = tryStart + tryLength; + this.handlerStart = handlerStart; + this.handlerEnd = handlerStart + handlerLength; + this.classToken = classToken; + } + } + + private static final String ONE_BYTE = + "00 nop n;01 break n;02 ldarg.0 n;03 ldarg.1 n;04 ldarg.2 n;05 ldarg.3 n;06 ldloc.0 n;07 ldloc.1 n;" + + "08 ldloc.2 n;09 ldloc.3 n;0A stloc.0 n;0B stloc.1 n;0C stloc.2 n;0D stloc.3 n;0E ldarg.s v1;" + + "0F ldarga.s v1;10 starg.s v1;11 ldloc.s v1;12 ldloca.s v1;13 stloc.s v1;14 ldnull n;15 ldc.i4.m1 n;" + + "16 ldc.i4.0 n;17 ldc.i4.1 n;18 ldc.i4.2 n;19 ldc.i4.3 n;1A ldc.i4.4 n;1B ldc.i4.5 n;1C ldc.i4.6 n;" + + "1D ldc.i4.7 n;1E ldc.i4.8 n;1F ldc.i4.s c1;20 ldc.i4 i4;21 ldc.i8 i8;22 ldc.r4 r4;23 ldc.r8 r8;" + + "25 dup n;26 pop n;27 jmp t;28 call t;29 calli t;2A ret n;2B br.s b1;2C brfalse.s b1;2D brtrue.s b1;" + + "2E beq.s b1;2F bge.s b1;30 bgt.s b1;31 ble.s b1;32 blt.s b1;33 bne.un.s b1;34 bge.un.s b1;" + + "35 bgt.un.s b1;36 ble.un.s b1;37 blt.un.s b1;38 br b4;39 brfalse b4;3A brtrue b4;3B beq b4;3C bge b4;" + + "3D bgt b4;3E ble b4;3F blt b4;40 bne.un b4;41 bge.un b4;42 bgt.un b4;43 ble.un b4;44 blt.un b4;" + + "45 switch s;46 ldind.i1 n;47 ldind.u1 n;48 ldind.i2 n;49 ldind.u2 n;4A ldind.i4 n;4B ldind.u4 n;" + + "4C ldind.i8 n;4D ldind.i n;4E ldind.r4 n;4F ldind.r8 n;50 ldind.ref n;51 stind.ref n;52 stind.i1 n;" + + "53 stind.i2 n;54 stind.i4 n;55 stind.i8 n;56 stind.r4 n;57 stind.r8 n;58 add n;59 sub n;5A mul n;" + + "5B div n;5C div.un n;5D rem n;5E rem.un n;5F and n;60 or n;61 xor n;62 shl n;63 shr n;64 shr.un n;" + + "65 neg n;66 not n;67 conv.i1 n;68 conv.i2 n;69 conv.i4 n;6A conv.i8 n;6B conv.r4 n;6C conv.r8 n;" + + "6D conv.u4 n;6E conv.u8 n;6F callvirt t;70 cpobj t;71 ldobj t;72 ldstr t;73 newobj t;74 castclass t;" + + "75 isinst t;76 conv.r.un n;79 unbox t;7A throw n;7B ldfld t;7C ldflda t;7D stfld t;7E ldsfld t;" + + "7F ldsflda t;80 stsfld t;81 stobj t;82 conv.ovf.i1.un n;83 conv.ovf.i2.un n;84 conv.ovf.i4.un n;" + + "85 conv.ovf.i8.un n;86 conv.ovf.u1.un n;87 conv.ovf.u2.un n;88 conv.ovf.u4.un n;89 conv.ovf.u8.un n;" + + "8A conv.ovf.i.un n;8B conv.ovf.u.un n;8C box t;8D newarr t;8E ldlen n;8F ldelema t;90 ldelem.i1 n;" + + "91 ldelem.u1 n;92 ldelem.i2 n;93 ldelem.u2 n;94 ldelem.i4 n;95 ldelem.u4 n;96 ldelem.i8 n;" + + "97 ldelem.i n;98 ldelem.r4 n;99 ldelem.r8 n;9A ldelem.ref n;9B stelem.i n;9C stelem.i1 n;" + + "9D stelem.i2 n;9E stelem.i4 n;9F stelem.i8 n;A0 stelem.r4 n;A1 stelem.r8 n;A2 stelem.ref n;" + + "A3 ldelem t;A4 stelem t;A5 unbox.any t;B3 conv.ovf.i1 n;B4 conv.ovf.u1 n;B5 conv.ovf.i2 n;" + + "B6 conv.ovf.u2 n;B7 conv.ovf.i4 n;B8 conv.ovf.u4 n;B9 conv.ovf.i8 n;BA conv.ovf.u8 n;C2 refanyval t;" + + "C3 ckfinite n;C6 mkrefany t;D0 ldtoken t;D1 conv.u2 n;D2 conv.u1 n;D3 conv.i n;D4 conv.ovf.i n;" + + "D5 conv.ovf.u n;D6 add.ovf n;D7 add.ovf.un n;D8 mul.ovf n;D9 mul.ovf.un n;DA sub.ovf n;" + + "DB sub.ovf.un n;DC endfinally n;DD leave b4;DE leave.s b1;DF stind.i n;E0 conv.u n"; + + private static final String TWO_BYTE = + "00 arglist n;01 ceq n;02 cgt n;03 cgt.un n;04 clt n;05 clt.un n;06 ldftn t;07 ldvirtftn t;09 ldarg v2;" + + "0A ldarga v2;0B starg v2;0C ldloc v2;0D ldloca v2;0E stloc v2;0F localloc n;11 endfilter n;" + + "12 unaligned. c1;13 volatile. n;14 tail. n;15 initobj t;16 constrained. t;17 cpblk n;18 initblk n;" + + "19 no. c1;1A rethrow n;1C sizeof t;1D refanytype n;1E readonly. n"; + + private static final String[][] ONE = parseTable(ONE_BYTE); + private static final String[][] TWO = parseTable(TWO_BYTE); + private static final Map CANONICAL = new HashMap(); + + private static String[][] parseTable(String spec) { + String[][] table = new String[256][]; + for (String entry : spec.split(";")) { + String[] parts = entry.split(" "); + table[Integer.parseInt(parts[0], 16)] = new String[] {parts[1], parts[2]}; + } + return table; + } + + /// Splits a compact opcode name into the general operation and the + /// operand the name implies: `ldloc.2` is `ldloc` 2, `br.s` is `br`. + private static Object[] canonical(String raw) { + Object[] cached = CANONICAL.get(raw); + if (cached != null) { + return cached; + } + String op = raw; + Integer implied = null; + if ("ldc.i4.m1".equals(raw)) { + op = "ldc.i4"; + implied = Integer.valueOf(-1); + } else if ("ldc.i4.s".equals(raw)) { + op = "ldc.i4"; + } else { + int dot = raw.lastIndexOf('.'); + String head = dot < 0 ? raw : raw.substring(0, dot); + String tail = dot < 0 ? "" : raw.substring(dot + 1); + boolean digit = tail.length() == 1 && tail.charAt(0) >= '0' && tail.charAt(0) <= '9'; + if (digit && ("ldarg".equals(head) || "ldloc".equals(head) || "stloc".equals(head) + || "ldc.i4".equals(head))) { + op = head; + implied = Integer.valueOf(tail.charAt(0) - '0'); + } else if ("s".equals(tail)) { + op = head; + } + } + Object[] out = new Object[] {op.intern(), implied}; + CANONICAL.put(raw, out); + return out; + } + + public final CilType[] locals; + public final List instructions = new ArrayList(); + public final List clauses = new ArrayList(); + private final Map indexByOffset = new HashMap(); + + MethodBody(CilAssembly assembly, int rva) { + MetadataReader md = assembly.metadata(); + int at = md.rvaToOffset(rva); + int first = md.u1(at); + int code; + int codeSize; + boolean moreSections = false; + if ((first & 3) == 2) { + // Tiny header: no locals, no clauses, at most 8 stack slots. + locals = new CilType[0]; + codeSize = first >> 2; + code = at + 1; + } else { + int flags = md.u2(at); + int headerSize = (flags >> 12) * 4; + codeSize = md.i4(at + 4); + int localsToken = md.i4(at + 8); + moreSections = (flags & 0x08) != 0; + locals = localsToken == 0 ? new CilType[0] : assembly.localsFromToken(localsToken); + code = at + headerSize; + } + decode(md, code, codeSize); + if (moreSections) { + readClauses(md, (code + codeSize + 3) & ~3); + } + } + + private void decode(MetadataReader md, int code, int codeSize) { + int p = 0; + while (p < codeSize) { + int start = p; + int b = md.u1(code + p++); + String[] def; + if (b == 0xFE) { + def = TWO[md.u1(code + p++)]; + } else { + def = ONE[b]; + } + if (def == null) { + throw new CilFormatException(md.source() + ": unknown opcode 0x" + Integer.toHexString(b) + + " at IL_" + Integer.toHexString(start)); + } + Object[] canon = canonical(def[0]); + String op = (String) canon[0]; + int operand = canon[1] == null ? 0 : ((Integer) canon[1]).intValue(); + long longOperand = 0; + double doubleOperand = 0; + int[] targets = null; + String kind = def[1]; + if ("v1".equals(kind)) { + operand = md.u1(code + p); + p += 1; + } else if ("v2".equals(kind)) { + operand = md.u2(code + p); + p += 2; + } else if ("c1".equals(kind)) { + operand = (byte) md.u1(code + p); + p += 1; + } else if ("i4".equals(kind) || "t".equals(kind)) { + operand = md.i4(code + p); + p += 4; + } else if ("i8".equals(kind)) { + longOperand = md.i8(code + p); + p += 8; + } else if ("r4".equals(kind)) { + doubleOperand = Float.intBitsToFloat(md.i4(code + p)); + p += 4; + } else if ("r8".equals(kind)) { + doubleOperand = Double.longBitsToDouble(md.i8(code + p)); + p += 8; + } else if ("b1".equals(kind)) { + int delta = (byte) md.u1(code + p); + p += 1; + operand = p + delta; + } else if ("b4".equals(kind)) { + int delta = md.i4(code + p); + p += 4; + operand = p + delta; + } else if ("s".equals(kind)) { + int count = md.i4(code + p); + p += 4; + targets = new int[count]; + int base = p + count * 4; + for (int i = 0; i < count; i++) { + targets[i] = base + md.i4(code + p); + p += 4; + } + } + indexByOffset.put(Integer.valueOf(start), Integer.valueOf(instructions.size())); + instructions.add(new Instr(start, p, op, operand, longOperand, doubleOperand, targets)); + } + } + + private void readClauses(MetadataReader md, int at) { + while (true) { + int kind = md.u1(at); + if ((kind & 0x01) == 0) { + throw new CilFormatException(md.source() + ": unknown method data section 0x" + + Integer.toHexString(kind)); + } + boolean fat = (kind & 0x40) != 0; + int size; + if (fat) { + size = md.u1(at + 1) | (md.u1(at + 2) << 8) | (md.u1(at + 3) << 16); + for (int c = at + 4; c + 24 <= at + size; c += 24) { + clauses.add(new Clause(md.i4(c), md.i4(c + 4), md.i4(c + 8), md.i4(c + 12), md.i4(c + 16), + md.i4(c + 20))); + } + } else { + size = md.u1(at + 1); + for (int c = at + 4; c + 12 <= at + size; c += 12) { + clauses.add(new Clause(md.u2(c), md.u2(c + 2), md.u1(c + 4), md.u2(c + 5), md.u1(c + 7), + md.i4(c + 8))); + } + } + if ((kind & 0x80) == 0) { + return; + } + at = (at + size + 3) & ~3; + } + } + + /// Position in [#instructions] of the instruction at an IL offset. + public int indexOf(int offset) { + Integer index = indexByOffset.get(Integer.valueOf(offset)); + if (index == null) { + throw new CilFormatException("no instruction starts at IL_" + Integer.toHexString(offset)); + } + return index.intValue(); + } +} diff --git a/maven/cil-translator/src/main/java/com/codename1/cil/metadata/PortablePdb.java b/maven/cil-translator/src/main/java/com/codename1/cil/metadata/PortablePdb.java new file mode 100644 index 00000000000..48170a6ed20 --- /dev/null +++ b/maven/cil-translator/src/main/java/com/codename1/cil/metadata/PortablePdb.java @@ -0,0 +1,375 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.cil.metadata; + +import java.nio.charset.StandardCharsets; + +/// The part of a portable PDB that says which line of which source file an +/// instruction came from. +/// +/// A portable PDB is the file the C# compiler writes beside an assembly +/// under `portable`. It is laid out as an assembly's +/// own metadata is (ECMA-335, II.24): the same root, the same heaps and a +/// table stream -- holding tables of its own, numbered from 0x30, in place +/// of the type system's. Two of them are read here: +/// +/// - `Document` (0x30): a source file, its name a blob of parts. +/// - `MethodDebugInformation` (0x31): one row for each row of the +/// assembly's `MethodDef` table, in the same order, with the method's +/// *sequence points* -- the places a debugger can stop, each an IL offset +/// and the source range it stands for. +/// +/// Those are the first two tables of the stream and neither has a column +/// that refers to a later one, so nothing else needs to be understood to +/// find them. The format is public: "Portable PDB v1.0: Format +/// Specification", in the `dotnet/runtime` documentation. +/// +/// It is only ever a help. A file that is absent, or is not what this +/// expects, gives no locations and no error: the translation it would have +/// described is unchanged. +public final class PortablePdb { + private static final int DOCUMENT = 0x30; + private static final int METHOD_DEBUG_INFORMATION = 0x31; + /// The line a compiler gives a sequence point that stands for no source. + private static final int HIDDEN_LINE = 0xFEEFEE; + + private final byte[] data; + private int stringsEnd; + private int blobOffset; + private int blobEnd; + private boolean wideBlobs; + private int documents; + private int documentsAt; + private int documentRow; + private int methods; + private int methodsAt; + private int methodRow; + private String[] documentNames; + private int at; + + private PortablePdb(byte[] data) { + this.data = data; + } + + /// Reads a PDB, or answers null for bytes that are not one. + public static PortablePdb read(byte[] data) { + PortablePdb pdb = new PortablePdb(data); + try { + return pdb.readRoot() ? pdb : null; + } catch (IndexOutOfBoundsException e) { + // Truncated, or not laid out as expected: no locations. + return null; + } + } + + private int u1(int p) { + return data[p] & 0xFF; + } + + private int u2(int p) { + return (data[p] & 0xFF) | ((data[p + 1] & 0xFF) << 8); + } + + private int i4(int p) { + return (data[p] & 0xFF) | ((data[p + 1] & 0xFF) << 8) | ((data[p + 2] & 0xFF) << 16) + | ((data[p + 3] & 0xFF) << 24); + } + + private boolean readRoot() { + if (data.length < 32 || i4(0) != 0x424A5342) { + return false; + } + int p = 16 + i4(12); + int streams = u2(p + 2); + p += 4; + int tables = -1; + boolean pdbStream = false; + for (int i = 0; i < streams; i++) { + int offset = i4(p); + int size = i4(p + 4); + p += 8; + int nameStart = p; + while (data[p] != 0) { + p++; + } + String name = new String(data, nameStart, p - nameStart, StandardCharsets.US_ASCII); + p = nameStart + ((p - nameStart + 4) & ~3); + if ("#~".equals(name)) { + tables = offset; + } else if ("#Blob".equals(name)) { + blobOffset = offset; + blobEnd = offset + size; + } else if ("#Strings".equals(name)) { + stringsEnd = offset + size; + } else if ("#Pdb".equals(name)) { + pdbStream = true; + } + } + if (tables < 0 || !pdbStream || blobEnd > data.length || stringsEnd > data.length) { + return false; + } + int heapSizes = u1(tables + 6); + boolean wideGuids = (heapSizes & 2) != 0; + wideBlobs = (heapSizes & 4) != 0; + long valid = (i4(tables + 8) & 0xFFFFFFFFL) | ((long) i4(tables + 12) << 32); + // A stand-alone PDB carries the debug tables alone. One merged into + // an assembly's own metadata would need every earlier table's row + // size, which is the other reader's business. + if ((valid & ((1L << DOCUMENT) - 1)) != 0) { + return false; + } + p = tables + 24; + int present = 0; + for (int t = DOCUMENT; t < 64; t++) { + if ((valid & (1L << t)) == 0) { + continue; + } + if (t == DOCUMENT) { + documents = i4(p); + } else if (t == METHOD_DEBUG_INFORMATION) { + methods = i4(p); + } + p += 4; + present++; + } + if (present == 0) { + return false; + } + int blob = wideBlobs ? 4 : 2; + int guid = wideGuids ? 4 : 2; + documentRow = blob + guid + blob + guid; + documentsAt = p; + methodRow = (documents < 65536 ? 2 : 4) + blob; + methodsAt = documentsAt + documents * documentRow; + if (methodsAt + methods * methodRow > data.length) { + return false; + } + documentNames = new String[documents + 1]; + return true; + } + + private int index(int p, boolean wide) { + return wide ? i4(p) : u2(p); + } + + /// An unsigned compressed integer (ECMA-335, II.23.2) at [#at], which + /// moves past it. + private int compressed() { + int first = u1(at); + if ((first & 0x80) == 0) { + at += 1; + return first; + } + if ((first & 0xC0) == 0x80) { + int v = ((first & 0x3F) << 8) | u1(at + 1); + at += 2; + return v; + } + int v = ((first & 0x1F) << 24) | (u1(at + 1) << 16) | (u1(at + 2) << 8) | u1(at + 3); + at += 4; + return v; + } + + /// A signed compressed integer: the sign is rotated into the lowest + /// bit, and how many bits the rest has depends on how many bytes the + /// number took. + private int signedCompressed() { + int start = at; + int raw = compressed(); + int value = raw >>> 1; + if ((raw & 1) == 0) { + return value; + } + int width = at - start; + return value - (width == 1 ? 0x40 : width == 2 ? 0x2000 : 0x10000000); + } + + /// The name of a document: a separator, then for each part of the path + /// the index of a blob holding it. Only the last part is kept -- the + /// rest is a directory of the machine that compiled it. + private String documentName(int row) { + if (row < 1 || row > documents) { + return null; + } + if (documentNames[row] != null) { + return documentNames[row]; + } + String last = documentText(row, false); + documentNames[row] = last; + return last; + } + + /// The name of a document, whole -- its parts joined by `/` whatever + /// the compiler's machine separated them with -- or its last part. + private String documentText(int row, boolean whole) { + int saved = at; + at = blobOffset + index(documentsAt + (row - 1) * documentRow, wideBlobs); + int length = compressed(); + int end = at + length; + int separator = length > 0 ? u1(at) : 0; + at++; + String last = ""; + StringBuilder path = new StringBuilder(); + while (at < end) { + int part = compressed(); + if (part == 0) { + continue; + } + int resume = at; + at = blobOffset + part; + int partLength = compressed(); + String text = new String(data, at, partLength, StandardCharsets.UTF_8); + at = resume; + if (text.length() > 0) { + last = text; + path.append(path.length() == 0 ? "" : "/").append(text); + } + } + at = saved; + if (whole) { + return path.toString().replace('\\', '/'); + } + if (separator == 0) { + // No separator: the name is one part, a whole path. + int slash = Math.max(last.lastIndexOf('/'), last.lastIndexOf('\\')); + last = slash >= 0 ? last.substring(slash + 1) : last; + } + return last; + } + + /// The source file a method was compiled from, as the compiler named + /// it: a path of the machine that compiled it, with `/` between its + /// parts. Null for a method with no source of its own, as a + /// constructor the compiler wrote is, and when the PDB does not say. + public String document(int methodDefRow) { + try { + if (methodDefRow < 1 || methodDefRow > methods) { + return null; + } + int row = methodsAt + (methodDefRow - 1) * methodRow; + int document = documents < 65536 ? u2(row) : i4(row); + if (document == 0) { + int blob = index(row + (documents < 65536 ? 2 : 4), wideBlobs); + if (blob == 0) { + return null; + } + // A method in several files: the header of its points, + // after the signature of the locals, names the first. + int saved = at; + at = blobOffset + blob; + compressed(); + compressed(); + document = compressed(); + at = saved; + } + if (document < 1 || document > documents) { + return null; + } + return documentText(document, true); + } catch (IndexOutOfBoundsException e) { + return null; + } + } + + /// Where in the source the instruction at an offset of a method came + /// from, as `File.cs:line`, or null when the PDB does not say. + /// `methodRow` is the method's row in the assembly's `MethodDef` table. + /// + /// The answer is the last sequence point at or before the offset that + /// stands for source, since an instruction belongs to the statement + /// that began before it; failing that -- an offset inside the code a + /// compiler puts ahead of the first statement -- the first that does. + public String location(int methodDefRow, int ilOffset) { + try { + return find(methodDefRow, ilOffset); + } catch (IndexOutOfBoundsException e) { + return null; + } + } + + private String find(int methodDefRow, int ilOffset) { + if (methodDefRow < 1 || methodDefRow > methods) { + return null; + } + int row = methodsAt + (methodDefRow - 1) * methodRow; + int document = documents < 65536 ? u2(row) : i4(row); + int blob = index(row + (documents < 65536 ? 2 : 4), wideBlobs); + if (blob == 0) { + return null; + } + at = blobOffset + blob; + int length = compressed(); + int end = at + length; + // The header: the signature of the locals, then the document when + // the row itself names none because the method spans several. + compressed(); + if (document == 0) { + document = compressed(); + } + String best = null; + String first = null; + int offset = 0; + int line = 0; + int column = 0; + boolean firstRecord = true; + boolean firstVisible = true; + while (at < end) { + int deltaOffset = compressed(); + if (!firstRecord && deltaOffset == 0) { + // Not a point: the points that follow are in another file. + document = compressed(); + continue; + } + firstRecord = false; + offset += deltaOffset; + int deltaLines = compressed(); + int deltaColumns = deltaLines == 0 ? compressed() : signedCompressed(); + if (deltaLines == 0 && deltaColumns == 0) { + // A hidden point: code that stands for no source. + continue; + } + if (firstVisible) { + line = compressed(); + column = compressed(); + firstVisible = false; + } else { + line += signedCompressed(); + column += signedCompressed(); + } + if (line == HIDDEN_LINE) { + continue; + } + String here = documentName(document); + here = here == null ? null : here + ":" + line; + if (first == null) { + first = here; + } + if (offset <= ilOffset) { + best = here; + } else { + break; + } + } + return best != null ? best : first; + } +} diff --git a/maven/cil-translator/src/main/java/com/codename1/cil/metadata/SignatureParser.java b/maven/cil-translator/src/main/java/com/codename1/cil/metadata/SignatureParser.java new file mode 100644 index 00000000000..d7c350952c3 --- /dev/null +++ b/maven/cil-translator/src/main/java/com/codename1/cil/metadata/SignatureParser.java @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.cil.metadata; + +/// Decodes the signature blobs of ECMA-335 II.23.2: field, method, local +/// variable and generic instantiation signatures, and the types inside them. +final class SignatureParser { + private final CilAssembly assembly; + private final MetadataReader md; + private int at; + + SignatureParser(CilAssembly assembly, int at) { + this.assembly = assembly; + this.md = assembly.metadata(); + this.at = at; + } + + private int compressed() { + int first = md.u1(at); + if ((first & 0x80) == 0) { + at += 1; + return first; + } + if ((first & 0xC0) == 0x80) { + int value = ((first & 0x3F) << 8) | md.u1(at + 1); + at += 2; + return value; + } + int value = ((first & 0x1F) << 24) | (md.u1(at + 1) << 16) | (md.u1(at + 2) << 8) | md.u1(at + 3); + at += 4; + return value; + } + + private int typeDefOrRefToken() { + return md.codedToken(MetadataReader.TYPE_DEF_OR_REF, compressed()); + } + + private CilFormatException bad(String what) { + return new CilFormatException(md.source() + ": " + what); + } + + // A signature states outright whether a named type is a class or a value + // type, so nothing has to be resolved to read one. + private CilType named(boolean valueType) { + int token = typeDefOrRefToken(); + if (MetadataReader.tokenTable(token) == MetadataReader.TYPE_SPEC) { + return assembly.typeFromToken(token); + } + String fullName = assembly.typeNameFromToken(token); + CilType builtin = CilType.builtin(fullName); + return builtin != null ? builtin : CilType.named(fullName, valueType); + } + + CilType type() { + while (true) { + int element = md.u1(at++); + switch (element) { + case 0x01: + return CilType.VOID; + case 0x02: + return CilType.BOOLEAN; + case 0x03: + return CilType.CHAR; + case 0x04: + return CilType.I1; + case 0x05: + return CilType.U1; + case 0x06: + return CilType.I2; + case 0x07: + return CilType.U2; + case 0x08: + return CilType.I4; + case 0x09: + return CilType.U4; + case 0x0A: + return CilType.I8; + case 0x0B: + return CilType.U8; + case 0x0C: + return CilType.R4; + case 0x0D: + return CilType.R8; + case 0x0E: + return CilType.STRING; + case 0x0F: + return CilType.pointer(type()); + case 0x10: + return CilType.byRef(type()); + case 0x11: + return named(true); + case 0x12: + return named(false); + case 0x13: + return CilType.var(compressed()); + case 0x14: { + CilType elementType = type(); + int rank = compressed(); + int sizes = compressed(); + for (int i = 0; i < sizes; i++) { + compressed(); + } + int lowerBounds = compressed(); + for (int i = 0; i < lowerBounds; i++) { + compressed(); + } + return CilType.array(elementType, rank); + } + case 0x15: { + int definitionKind = md.u1(at++); + CilType definition = named(definitionKind == 0x11); + CilType[] args = new CilType[compressed()]; + for (int i = 0; i < args.length; i++) { + args[i] = type(); + } + return CilType.genericInst(definition, args); + } + case 0x16: + return CilType.TYPEDBYREF; + case 0x18: + return CilType.I; + case 0x19: + return CilType.U; + case 0x1B: + methodSig(); + return CilType.FNPTR; + case 0x1C: + return CilType.OBJECT; + case 0x1D: + return CilType.szArray(type()); + case 0x1E: + return CilType.mvar(compressed()); + case 0x1F: + case 0x20: + // A custom modifier (modreq/modopt) qualifies the type + // that follows it and does not change what that type is. + compressed(); + break; + case 0x41: + case 0x45: + // Sentinel and pinned: markers in front of a type. + break; + default: + throw bad("unknown element type 0x" + Integer.toHexString(element) + " in a signature"); + } + } + } + + CilType fieldSig() { + int lead = md.u1(at++); + if (lead != 0x06) { + throw bad("expected a field signature, found 0x" + Integer.toHexString(lead)); + } + return type(); + } + + CilAssembly.MethodSig methodSig() { + int lead = md.u1(at++); + boolean hasThis = (lead & 0x20) != 0; + int genericParams = (lead & 0x10) != 0 ? compressed() : 0; + CilType[] params = new CilType[compressed()]; + CilType returnType = type(); + for (int i = 0; i < params.length; i++) { + params[i] = type(); + } + return new CilAssembly.MethodSig(hasThis, genericParams, returnType, params); + } + + CilType[] locals() { + int lead = md.u1(at++); + if (lead != 0x07) { + throw bad("expected a local variable signature, found 0x" + Integer.toHexString(lead)); + } + CilType[] out = new CilType[compressed()]; + for (int i = 0; i < out.length; i++) { + out[i] = type(); + } + return out; + } + + CilType[] methodSpec() { + int lead = md.u1(at++); + if (lead != 0x0A) { + throw bad("expected a generic instantiation, found 0x" + Integer.toHexString(lead)); + } + CilType[] out = new CilType[compressed()]; + for (int i = 0; i < out.length; i++) { + out[i] = type(); + } + return out; + } +} diff --git a/maven/cil-translator/src/main/java/com/codename1/cil/metadata/Universe.java b/maven/cil-translator/src/main/java/com/codename1/cil/metadata/Universe.java new file mode 100644 index 00000000000..b752c0626e9 --- /dev/null +++ b/maven/cil-translator/src/main/java/com/codename1/cil/metadata/Universe.java @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.cil.metadata; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/// Every assembly a translation can see: the ones being translated and the +/// reference assemblies they were compiled against. +/// +/// Types are looked up by full name alone. Which assembly a reference claims +/// the type lives in is ignored, because a compile-time reference names a +/// facade or a reference assembly, never the place the code ends up at run time. +public final class Universe { + private final List assemblies = new ArrayList(); + private final Map types = new HashMap(); + + /// The debug information of an assembly is the file of the same name + /// with `.pdb` where the assembly has `.dll` or `.exe`. + private static File pdbBeside(File assembly) { + String name = assembly.getName(); + int dot = name.lastIndexOf('.'); + return new File(assembly.getAbsoluteFile().getParentFile(), + (dot > 0 ? name.substring(0, dot) : name) + ".pdb"); + } + + /// Loads a set of assemblies together. The first to define a name wins, + /// so list the assemblies being translated before their references. + public List load(List files) throws IOException { + List loaded = new ArrayList(); + for (File file : files) { + CilAssembly assembly = new CilAssembly(Files.readAllBytes(file.toPath()), file.getName()); + File pdb = pdbBeside(file); + if (pdb.isFile()) { + assembly.debugInformation(PortablePdb.read(Files.readAllBytes(pdb.toPath()))); + } + assembly.load(this); + for (CilAssembly.TypeDef t : assembly.types()) { + if (!types.containsKey(t.fullName())) { + types.put(t.fullName(), t); + } + } + loaded.add(assembly); + } + for (CilAssembly assembly : loaded) { + assembly.linkBases(); + } + for (CilAssembly assembly : loaded) { + assembly.link(); + } + for (CilAssembly assembly : loaded) { + assembly.linkOverrides(); + } + assemblies.addAll(loaded); + return loaded; + } + + public List assemblies() { + return Collections.unmodifiableList(assemblies); + } + + public CilAssembly.TypeDef find(String fullName) { + return types.get(fullName); + } + + public CilAssembly.TypeDef require(String fullName, String neededBy) { + CilAssembly.TypeDef found = types.get(fullName); + if (found == null) { + throw new CilFormatException(neededBy + ": type " + fullName + + " is not defined by any loaded assembly (is a reference assembly missing?)"); + } + return found; + } + + /// The definition behind a type, or null for one that has none of its + /// own: an array, a type variable, a pointer. + public CilAssembly.TypeDef definitionOf(CilType type) { + String name = type.typeName(); + return name == null ? null : types.get(name); + } + + /// Finds the method a reference names, looking up the inheritance chain + /// when the named type only inherits it. Signatures are compared as + /// declared, type variables included, which is how a reference spells them. + public CilAssembly.MethodDef resolve(CilAssembly.MethodRef ref) { + if (ref.def != null) { + return ref.def; + } + CilAssembly.TypeDef type = definitionOf(ref.declaringType); + while (type != null) { + for (CilAssembly.MethodDef m : type.methods) { + if (m.name.equals(ref.name) && m.sig.sameShape(ref.sig)) { + return m; + } + } + type = type.baseType == null ? null : definitionOf(type.baseType); + } + return null; + } + + public CilAssembly.FieldDef resolve(CilAssembly.FieldRef ref) { + if (ref.def != null) { + return ref.def; + } + CilAssembly.TypeDef type = definitionOf(ref.declaringType); + while (type != null) { + CilAssembly.FieldDef f = type.field(ref.name); + if (f != null) { + return f; + } + type = type.baseType == null ? null : definitionOf(type.baseType); + } + return null; + } + + /// True when `type` is `ancestor` or derives from it, through base + /// classes or implemented interfaces. + public boolean derivesFrom(String type, String ancestor) { + if (type.equals(ancestor) || "System.Object".equals(ancestor)) { + return true; + } + CilAssembly.TypeDef def = types.get(type); + if (def == null) { + return false; + } + for (CilType i : def.interfaces) { + String name = i.typeName(); + if (name != null && derivesFrom(name, ancestor)) { + return true; + } + } + String base = def.baseType == null ? null : def.baseType.typeName(); + return base != null && derivesFrom(base, ancestor); + } +} diff --git a/maven/cil-translator/src/main/java/com/codename1/cil/tools/MetadataDump.java b/maven/cil-translator/src/main/java/com/codename1/cil/tools/MetadataDump.java new file mode 100644 index 00000000000..2db148b5f14 --- /dev/null +++ b/maven/cil-translator/src/main/java/com/codename1/cil/tools/MetadataDump.java @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.cil.tools; + +import com.codename1.cil.metadata.CilAssembly; +import com.codename1.cil.metadata.CilType; +import com.codename1.cil.metadata.MethodBody; +import com.codename1.cil.metadata.Universe; +import java.io.File; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.List; + +/// Prints what the metadata reader sees in an assembly, one line per type, +/// field and method. +/// +/// The format is shared with `scripts/unity-compat-samples/metadata-dump`, +/// which prints the same lines from .NET's own `System.Reflection.Metadata`. +/// Comparing the two outputs is how the reader is checked against an +/// independent implementation, so change both together. +public final class MetadataDump { + private MetadataDump() { + } + + public static void main(String[] args) throws Exception { + if (args.length == 0) { + System.err.println("usage: MetadataDump [reference assembly ...]"); + System.exit(2); + } + List files = new ArrayList(); + boolean il = false; + for (String arg : args) { + if ("--il".equals(arg)) { + il = true; + } else { + files.add(new File(arg)); + } + } + Universe universe = new Universe(); + CilAssembly assembly = universe.load(files).get(0); + // Flushed, not closed: closing it would close System.out. + PrintStream out = new PrintStream(System.out, false, "UTF-8"); // NOPMD CloseResource + if (il) { + dumpIl(assembly, out); + } else { + dump(assembly, out); + } + out.flush(); + } + + /// Lists every method body with its tokens resolved. A reading aid for + /// whoever works on the translator; nothing compares this output. + static void dumpIl(CilAssembly assembly, PrintStream out) { + for (CilAssembly.TypeDef t : assembly.types()) { + for (CilAssembly.MethodDef m : t.methods) { + out.println(m); + if (!m.hasBody()) { + continue; + } + MethodBody body = m.body(); + for (int i = 0; i < body.locals.length; i++) { + out.println(" local " + i + " : " + body.locals[i]); + } + for (MethodBody.Clause c : body.clauses) { + out.println(" clause flags=" + c.flags + " try=" + Integer.toHexString(c.tryStart) + ".." + + Integer.toHexString(c.tryEnd) + " handler=" + Integer.toHexString(c.handlerStart) + + ".." + Integer.toHexString(c.handlerEnd) + + (c.flags == MethodBody.Clause.CATCH ? " " + assembly.typeFromToken(c.classToken) : "")); + } + for (MethodBody.Instr in : body.instructions) { + out.println(" " + in + operand(assembly, in)); + } + } + } + out.flush(); + } + + private static String operand(CilAssembly assembly, MethodBody.Instr in) { + String op = in.op; + if ("call".equals(op) || "callvirt".equals(op) || "newobj".equals(op) || "ldftn".equals(op) + || "ldvirtftn".equals(op)) { + CilAssembly.MethodRef ref = assembly.methodRef(in.operand); + String inst = ""; + if (ref.methodArgs != null) { + inst = " <" + java.util.Arrays.toString(ref.methodArgs) + ">"; + } + return " " + ref + inst; + } + if ("ldfld".equals(op) || "ldflda".equals(op) || "stfld".equals(op) || "ldsfld".equals(op) + || "ldsflda".equals(op) || "stsfld".equals(op)) { + CilAssembly.FieldRef ref = assembly.fieldRef(in.operand); + return " " + ref + " : " + ref.type; + } + if ("ldstr".equals(op)) { + return " \"" + assembly.userString(in.operand) + "\""; + } + if ("ldtoken".equals(op)) { + return assembly.isFieldToken(in.operand) ? " field " + assembly.fieldRef(in.operand) : " token"; + } + if ("newarr".equals(op) || "box".equals(op) || "unbox".equals(op) || "unbox.any".equals(op) + || "castclass".equals(op) || "isinst".equals(op) || "initobj".equals(op) || "constrained.".equals(op) + || "ldobj".equals(op) || "stobj".equals(op) || "cpobj".equals(op) || "ldelem".equals(op) + || "stelem".equals(op) || "ldelema".equals(op) || "sizeof".equals(op)) { + return " " + assembly.typeFromToken(in.operand); + } + if ("ldc.i8".equals(op)) { + return " " + in.longOperand; + } + if ("ldc.r4".equals(op) || "ldc.r8".equals(op)) { + return " " + in.doubleOperand; + } + if ("switch".equals(op)) { + StringBuilder sb = new StringBuilder(" ["); + for (int target : in.targets) { + sb.append(" IL_").append(Integer.toHexString(target)); + } + return sb.append(" ]").toString(); + } + if (op.startsWith("b") && !"box".equals(op) && !"break".equals(op) || "leave".equals(op)) { + return " IL_" + Integer.toHexString(in.operand); + } + if (op.startsWith("ld") || op.startsWith("st")) { + if ("ldarg".equals(op) || "ldarga".equals(op) || "starg".equals(op) || "ldloc".equals(op) + || "ldloca".equals(op) || "stloc".equals(op) || "ldc.i4".equals(op)) { + return " " + in.operand; + } + } + return ""; + } + + static void dump(CilAssembly assembly, PrintStream out) { + for (CilAssembly.TypeDef t : assembly.types()) { + out.println("T " + t.fullName() + " flags=" + Integer.toHexString(t.flags) + " base=" + + (t.baseType == null ? "-" : plain(t.baseType)) + " generic=" + t.genericParamCount); + for (CilType i : t.interfaces) { + out.println(" I " + plain(i)); + } + for (CilAssembly.FieldDef f : t.fields) { + out.println(" F " + f.name + " flags=" + Integer.toHexString(f.flags) + " : " + f.type); + } + for (CilAssembly.MethodDef m : t.methods) { + String code = " code=-1 locals=0 clauses=0"; + if (m.hasBody()) { + MethodBody body = m.body(); + int size = 0; + if (!body.instructions.isEmpty()) { + size = body.instructions.get(body.instructions.size() - 1).next; + } + code = " code=" + size + " locals=" + body.locals.length + " clauses=" + body.clauses.size(); + } + out.println(" M " + m.name + " flags=" + Integer.toHexString(m.flags) + " " + m.sig + code); + } + } + out.flush(); + } + + // A base type or an implemented interface is named by a bare token, which + // carries no class-or-value-type marker; print the name alone so that the + // line does not depend on resolving it. + private static String plain(CilType type) { + String text = type.toString(); + return text.startsWith("valuetype ") ? text.substring("valuetype ".length()) : text; + } +} diff --git a/maven/cil-translator/src/main/java/com/codename1/cil/translate/GenericStatics.java b/maven/cil-translator/src/main/java/com/codename1/cil/translate/GenericStatics.java new file mode 100644 index 00000000000..d076dcea394 --- /dev/null +++ b/maven/cil-translator/src/main/java/com/codename1/cil/translate/GenericStatics.java @@ -0,0 +1,855 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.cil.translate; + +import com.codename1.cil.metadata.CilAssembly; +import com.codename1.cil.metadata.CilAssembly.FieldDef; +import com.codename1.cil.metadata.CilAssembly.FieldRef; +import com.codename1.cil.metadata.CilAssembly.MethodDef; +import com.codename1.cil.metadata.CilAssembly.MethodRef; +import com.codename1.cil.metadata.CilAssembly.TypeDef; +import com.codename1.cil.metadata.CilType; +import com.codename1.cil.metadata.MetadataReader; +import com.codename1.cil.metadata.MethodBody.Instr; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; + +/// The static state of generic types. +/// +/// Generics are erased, so ``Cache`1`` is one JVM class whatever it is a +/// cache of. C# gives every constructed type its own static fields and runs +/// the static constructor once for each: `Cache.Value` and +/// `Cache.Value` are two variables. One Java static for both hands +/// one manager the other's instance in the commonest shape there is, +/// `Singleton`, and on ParparVM the cast that follows is unchecked. +/// +/// So the static fields of a generic type are not Java statics. They are +/// the instance fields of a *context*: an object of the generated class +/// `Type$$S`, of which there is one per constructed type. Which one is +/// settled where the type arguments are known: +/// +/// - Code that names a closed type -- `Cache.Value` anywhere outside +/// the generic type -- reads the context from a static field of a class +/// generated for that one constructed type, `Type$$S$`. That +/// class's own initializer makes the context and runs the C# static +/// constructor on it, so the JVM's once-per-class rule is .NET's +/// once-per-constructed-type rule and nothing is looked up at run time. +/// - A static method of the generic type takes its context as one more +/// parameter, after every other; so does a constructor, which keeps it in +/// a field of the object for the instance methods to read. +/// - Code that names a type built from its own type variables, as +/// `Outer.Inner` naming `Outer` does, asks its own context: the +/// context class declares one method per such type, and each +/// `Type$$S$` answers it with the context of the type those +/// arguments make of it. A type that needs one for that reason alone has +/// a context too, with no fields in it. +/// - A generic *method* that names a type built from its own type variables +/// takes a context of its own in the same way (`Type$$M`). +/// +/// A context also answers `typeof(T)`, and the class a type variable +/// travels as to a generic method of the runtime, so `FindObjectOfType()` +/// inside `Singleton` finds a `T`. It answers, too, for the constructed +/// types its code builds from its own variables and that erasure would +/// otherwise make one: `typeof(List)`, and the closed type of a +/// delegate it makes, `Action` (`Delegate.$typeKey`). Each is a +/// constant of the `Type$$S$` class, so making one costs no +/// more than a call. Code without a context still cannot tell them apart +/// (see `Delegate.$typeKey` for what that leaves). +/// +/// None of this touches a type without type parameters, or a generic type +/// with no static state that names none that has: when no generic type of +/// the program declares a static field, nothing is even scanned. +/// +/// What has no context is an error, never a shared variable: an instance +/// method of a generic *struct* (a struct is built without a constructor +/// call, so nothing could hand it one), and a virtual generic method that +/// would need a parameter its overrides and its interface do not have. +final class GenericStatics implements Opcodes { + /// The static field of a `Type$$S$` class that holds the context. + static final String INSTANCE = "$I"; + /// How many contexts deep one constructed type may reach for another + /// before the family is taken to be endless, as + /// `class G { static int n; int f() { return G>.n; } }` is. + private static final int DEPTH_LIMIT = 24; + + /// A reference to something that has a context: a generic type, or a + /// generic method when `method` is set, with the arguments the + /// reference gives it. Those may still name type variables of the code + /// the reference is in. + static final class Use { + final TypeDef type; + final MethodDef method; + final CilType[] typeArgs; + final CilType[] methodArgs; + + Use(TypeDef type, MethodDef method, CilType[] typeArgs, CilType[] methodArgs) { + this.type = type; + this.method = method; + this.typeArgs = typeArgs == null ? new CilType[0] : typeArgs; + this.methodArgs = methodArgs == null ? new CilType[0] : methodArgs; + } + + /// True when an argument names a type variable of a method. + boolean namesMethodVariable() { + return mentions(typeArgs, CilType.Kind.MVAR) || mentions(methodArgs, CilType.Kind.MVAR); + } + + /// True when an argument names any type variable: the context is + /// not known until the code around it knows its own. + boolean open() { + return namesMethodVariable() || mentions(typeArgs, CilType.Kind.VAR) + || mentions(methodArgs, CilType.Kind.VAR); + } + + Use substitute(CilType[] types, CilType[] methods) { + return new Use(type, method, substitute(typeArgs, types, methods), substitute(methodArgs, types, methods)); + } + + private static CilType[] substitute(CilType[] in, CilType[] types, CilType[] methods) { + CilType[] out = new CilType[in.length]; + for (int i = 0; i < in.length; i++) { + out[i] = in[i].substitute(types, methods); + } + return out; + } + + String arguments() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < typeArgs.length; i++) { + sb.append(i > 0 ? "," : "").append(typeArgs[i]); + } + if (method != null) { + sb.append(';'); + for (int i = 0; i < methodArgs.length; i++) { + sb.append(i > 0 ? "," : "").append(methodArgs[i]); + } + } + return sb.toString(); + } + + String key() { + return (method == null ? type.fullName() : method.toString()) + "<" + arguments() + ">"; + } + + /// As C# would write it, for a message. + @Override + public String toString() { + StringBuilder sb = new StringBuilder(type.fullName()); + sb.append('<'); + for (int i = 0; i < typeArgs.length; i++) { + sb.append(i > 0 ? "," : "").append(typeArgs[i]); + } + sb.append('>'); + if (method != null) { + sb.append("::").append(method.name).append('<'); + for (int i = 0; i < methodArgs.length; i++) { + sb.append(i > 0 ? "," : "").append(methodArgs[i]); + } + sb.append('>'); + } + return sb.toString(); + } + } + + private static boolean mentions(CilType[] types, CilType.Kind kind) { + if (types != null) { + for (CilType t : types) { + if (mentions(t, kind)) { + return true; + } + } + } + return false; + } + + /// True when a type names a type variable of a method. + static boolean namesMethodVariable(CilType type) { + return mentions(type, CilType.Kind.MVAR); + } + + private static boolean mentions(CilType type, CilType.Kind kind) { + if (type == null) { + return false; + } + return type.kind == kind || mentions(type.element, kind) || mentions(type.args, kind); + } + + private final Translator host; + private boolean analyzed; + private final Set keyedTypes = new HashSet(); + private final Set keyedMethods = new HashSet(); + private final Map> dependencies = new HashMap>(); + private final Map> openTypes = new HashMap>(); + /// The arguments each generated `...$` class stands for, to + /// notice two that were given one name. + private final Map closedClasses = new HashMap(); + private int depth; + private Use outermost; + + GenericStatics(Translator host) { + this.host = host; + } + + // --------------------------------------------------------------- analysis + + private static boolean isStaticStorage(FieldDef f) { + return f.isStatic() && !f.isLiteral() && f.rva == 0; + } + + private static boolean hasStaticState(TypeDef type) { + for (FieldDef f : type.fields) { + if (isStaticStorage(f)) { + return true; + } + } + return staticConstructor(type) != null; + } + + static MethodDef staticConstructor(TypeDef type) { + for (MethodDef m : type.methods) { + if (".cctor".equals(m.name)) { + return m; + } + } + return null; + } + + /// The classes the C# compiler writes to keep delegates it made once: + /// `<>c`, with the object every lambda that captures nothing is a + /// method of, and `<>O`, with the delegates of static method groups. + /// What they keep is the same for every constructed type unless the + /// code around them has a context to tell them apart by, so by + /// themselves they do not make a type one that needs a context -- or + /// every generic class with a lambda in it would pay for one. + private static boolean isDelegateCache(TypeDef type) { + return (type.name.startsWith("<>c") && !type.name.startsWith("<>c__DisplayClass")) + || type.name.startsWith("<>O"); + } + + private static boolean isCandidate(TypeDef type) { + return type.genericParamCount > 0 && !type.isEnum() && !type.isDelegate() && !type.isInterface(); + } + + private void analyze() { + if (analyzed) { + return; + } + analyzed = true; + List types = new ArrayList(); + List methods = new ArrayList(); + boolean any = false; + for (CilAssembly assembly : host.universe.assemblies()) { + for (TypeDef type : assembly.types()) { + if (!host.names.isTranslated(type.fullName()) || type.isInterface()) { + continue; + } + if (isCandidate(type)) { + types.add(type); + if (hasStaticState(type) && !isDelegateCache(type)) { + keyedTypes.add(type); + any = true; + } + } + for (MethodDef m : type.methods) { + if (m.genericParamCount > 0 && m.hasBody()) { + methods.add(m); + } + } + } + } + if (!any) { + return; + } + // Whatever names a type that has a context, through its own type + // variables, needs a context to find it by: until nothing changes. + boolean changed = true; + while (changed) { + changed = false; + for (TypeDef type : types) { + if (keyedTypes.contains(type)) { + continue; + } + if (isDelegateCache(type) && hasStaticState(type) && type.enclosing != null + && (keyedTypes.contains(type.enclosing) || declaresKeyedMethod(type.enclosing))) { + keyedTypes.add(type); + changed = true; + continue; + } + for (MethodDef m : type.methods) { + if (namesOpenUse(m, false)) { + keyedTypes.add(type); + changed = true; + break; + } + } + } + for (MethodDef m : methods) { + if (!keyedMethods.contains(m) && namesOpenUse(m, true)) { + if (m.isVirtual() || m.isAbstract()) { + throw new TranslationException(m + ": a virtual generic method, or one that implements an" + + " interface, uses " + firstOpenUse(m, true) + ", whose static state belongs to" + + " each constructed type; the method would need a parameter its overrides do not" + + " have. Make the method non-virtual, or keep that state outside the generic type"); + } + keyedMethods.add(m); + changed = true; + } + } + } + } + + private boolean declaresKeyedMethod(TypeDef type) { + for (MethodDef m : type.methods) { + if (keyedMethods.contains(m)) { + return true; + } + } + return false; + } + + private boolean namesOpenUse(MethodDef m, boolean ofMethod) { + return firstOpenUse(m, ofMethod) != null; + } + + private Use firstOpenUse(MethodDef m, boolean ofMethod) { + for (Use u : uses(m)) { + if (u.open() && u.namesMethodVariable() == ofMethod && !isSelf(u, m.owner, ofMethod ? m : null)) { + return u; + } + } + return null; + } + + /// Every reference in a method body to something that has a context. + private List uses(MethodDef m) { + List out = new ArrayList(); + if (!m.hasBody()) { + return out; + } + CilAssembly assembly = m.owner.assembly; + for (Instr ins : m.body().instructions) { + String op = ins.op; + if ("ldsfld".equals(op) || "ldsflda".equals(op) || "stsfld".equals(op)) { + add(out, fieldUse0(assembly.fieldRef(ins.operand))); + } else if ("call".equals(op) || "callvirt".equals(op) || "newobj".equals(op) || "ldftn".equals(op) + || "ldvirtftn".equals(op)) { + MethodRef ref = assembly.methodRef(ins.operand); + add(out, typeUse0(ref)); + add(out, methodUse0(ref)); + } + } + return out; + } + + private static void add(List out, Use u) { + if (u != null) { + out.add(u); + } + } + + private static CilType[] identity(int count, boolean ofMethod) { + CilType[] out = new CilType[count]; + for (int i = 0; i < count; i++) { + out[i] = ofMethod ? CilType.mvar(i) : CilType.var(i); + } + return out; + } + + private static boolean isIdentity(CilType[] args, CilType.Kind kind) { + for (int i = 0; i < args.length; i++) { + if (args[i].kind != kind || args[i].index != i) { + return false; + } + } + return true; + } + + /// True when a use inside `type` (and `method`, for a method's own + /// context) names that very thing with its own variables: the context + /// the code already has. + boolean isSelf(Use u, TypeDef type, MethodDef method) { + return u.type == type && u.method == method // NOPMD CompareObjectsWithEquals + && isIdentity(u.typeArgs, CilType.Kind.VAR) && isIdentity(u.methodArgs, CilType.Kind.MVAR); + } + + /// The arguments `target` has in a reference that names `from`, which + /// is `target` or a type deriving from it: C# may name a subclass where + /// it means an inherited static. + private CilType[] argumentsOf(CilType from, TypeDef target) { + CilType at = from; + for (int i = 0; i < 64 && at != null; i++) { + TypeDef def = host.universe.definitionOf(at); + if (def == null) { + break; + } + CilType[] args = at.kind == CilType.Kind.GENERICINST ? at.args : null; + if (def == target) { // NOPMD CompareObjectsWithEquals + return args != null ? args : identity(target.genericParamCount, false); + } + at = def.baseType == null ? null : def.baseType.substitute(args, null); + } + throw new TranslationException("cannot tell which " + target.fullName() + " a reference through " + from + + " means"); + } + + private Use fieldUse0(FieldRef ref) { + if (keyedTypes.isEmpty() || host.universe.definitionOf(ref.declaringType) == null) { + return null; + } + FieldDef f = host.universe.resolve(ref); + if (f == null || !isStaticStorage(f) || !keyedTypes.contains(f.owner)) { + return null; + } + return new Use(f.owner, null, argumentsOf(ref.declaringType, f.owner), null); + } + + private Use typeUse0(MethodRef ref) { + if (keyedTypes.isEmpty() || host.universe.definitionOf(ref.declaringType) == null) { + return null; + } + MethodDef m = host.universe.resolve(ref); + if (m == null || !takesTypeContext(m)) { + return null; + } + return new Use(m.owner, null, argumentsOf(ref.declaringType, m.owner), null); + } + + private Use methodUse0(MethodRef ref) { + if (keyedMethods.isEmpty() || host.universe.definitionOf(ref.declaringType) == null) { + return null; + } + MethodDef m = host.universe.resolve(ref); + if (m == null || !keyedMethods.contains(m)) { + return null; + } + return new Use(m.owner, m, m.owner.genericParamCount == 0 ? null : argumentsOf(ref.declaringType, m.owner), + ref.methodArgs != null ? ref.methodArgs : identity(m.genericParamCount, true)); + } + + // ---------------------------------------------------------------- queries + + /// True for a generic type whose static state is kept per constructed + /// type, in a context. + boolean hasContext(TypeDef type) { + analyze(); + return keyedTypes.contains(type); + } + + /// True for a generic method that takes a context of its own. + boolean hasContext(MethodDef method) { + analyze(); + return keyedMethods.contains(method); + } + + /// True for a method that is handed its type's context as a parameter: + /// a static method, and a constructor of a class. + boolean takesTypeContext(MethodDef m) { + analyze(); + return keyedTypes.contains(m.owner) + && (m.isStatic() || (".ctor".equals(m.name) && !m.owner.isValueType())); + } + + /// The context a static field lives in, or null for an ordinary static. + Use fieldUse(FieldRef ref) { + analyze(); + return fieldUse0(ref); + } + + /// The context a call must pass for the type of the method it names: + /// set for a static method or a constructor of a type that has one. + Use typeUse(MethodRef ref) { + analyze(); + return typeUse0(ref); + } + + /// The context a call must pass for the generic method it names. + Use methodUse(MethodRef ref) { + analyze(); + return methodUse0(ref); + } + + /// The class of a type's contexts. + String contextClass(TypeDef type) { + return host.names.className(type.fullName()) + "$$S"; + } + + /// The class of a generic method's contexts. + String contextClass(MethodDef method) { + return host.names.className(method.owner.fullName()) + "$$M" + method.owner.methods.indexOf(method); + } + + String contextClass(Use u) { + return u.method == null ? contextClass(u.type) : contextClass(u.method); + } + + /// The field of an object of a generic class that holds its context. + /// Named after the class, because a class deriving from it may have a + /// context -- and such a field -- of its own. + String contextField(TypeDef type) { + return "$s$" + host.names.className(type.fullName()).replace('/', '_'); + } + + /// What the hidden parameters add to a method's descriptor: its type's + /// context, then its own. + String hiddenParameters(MethodDef m) { + analyze(); + if (keyedTypes.isEmpty()) { + return ""; + } + String out = takesTypeContext(m) ? "L" + contextClass(m.owner) + ";" : ""; + return keyedMethods.contains(m) ? out + "L" + contextClass(m) + ";" : out; + } + + static String withHidden(String descriptor, String hidden) { + if (hidden.length() == 0) { + return descriptor; + } + int at = descriptor.lastIndexOf(')'); + return descriptor.substring(0, at) + hidden + descriptor.substring(at); + } + + /// The types built from a type's own variables that its code names: + /// what its context has to be able to answer for. + Map dependencies(TypeDef type) { + Map found = dependencies.get(type); + if (found == null) { + found = new LinkedHashMap(); + for (MethodDef m : type.methods) { + for (Use u : uses(m)) { + if (u.open() && !u.namesMethodVariable() && !isSelf(u, type, null)) { + found.put(u.key(), u); + } + } + } + checkNames(found, type.fullName()); + dependencies.put(type, found); + } + return found; + } + + Map dependencies(MethodDef method) { + Map found = dependencies.get(method); + if (found == null) { + found = new LinkedHashMap(); + for (Use u : uses(method)) { + if (u.namesMethodVariable() && !isSelf(u, method.owner, method)) { + found.put(u.key(), u); + } + } + checkNames(found, method.toString()); + dependencies.put(method, found); + } + return found; + } + + private static void checkNames(Map found, String where) { + Set names = new HashSet(); + for (Use u : found.values()) { + if (!names.add(getter(u))) { + throw new TranslationException(where + ": two of the generic types it names have one name once" + + " shortened (" + getter(u) + ")"); + } + } + } + + /// The constructed types built from a type's own variables whose closed + /// form its code needs at run time, by the method of its context that + /// answers each: see [#keyGetter(CilType)] and [#typeOfGetter(CilType)]. + /// A type's own are those that name no variable of a method; a + /// method's are every one its body names. + Map openTypes(TypeDef type) { + Map found = openTypes.get(type); + if (found == null) { + found = new LinkedHashMap(); + for (MethodDef m : type.methods) { + scanOpenTypes(m, false, found); + } + openTypes.put(type, found); + } + return found; + } + + Map openTypes(MethodDef method) { + Map found = openTypes.get(method); + if (found == null) { + found = new LinkedHashMap(); + scanOpenTypes(method, true, found); + openTypes.put(method, found); + } + return found; + } + + private void scanOpenTypes(MethodDef m, boolean ofMethod, Map out) { + if (!m.hasBody()) { + return; + } + CilAssembly assembly = m.owner.assembly; + for (Instr ins : m.body().instructions) { + CilType t = null; + boolean delegate = false; + if ("newobj".equals(ins.op)) { + MethodRef ref = assembly.methodRef(ins.operand); + TypeDef def = host.universe.definitionOf(ref.declaringType); + if (def != null && def.isDelegate() && ref.declaringType.kind == CilType.Kind.GENERICINST) { + t = ref.declaringType; + delegate = true; + } + } else if ("ldtoken".equals(ins.op) + && MetadataReader.tokenTable(ins.operand) == MetadataReader.TYPE_SPEC) { + CilType x = assembly.typeFromToken(ins.operand); + if (x.kind == CilType.Kind.GENERICINST) { + t = x; + } + } + if (t == null || !(mentions(t, CilType.Kind.VAR) || mentions(t, CilType.Kind.MVAR)) + || !ofMethod && mentions(t, CilType.Kind.MVAR)) { + continue; + } + out.put(delegate ? keyGetter(t) : typeOfGetter(t), t); + } + } + + /// The method of a context that answers the closed type of a delegate + /// type its code makes, as the string constant `Delegate.$typeKey` + /// holds -- the one code naming the closed type writes, so the two + /// are one interned object. + static String keyGetter(CilType open) { + return "$k" + mangle(open.toString()); + } + + /// The method of a context that answers `typeof` of a constructed type + /// its code names. + static String typeOfGetter(CilType open) { + return "$y" + mangle(open.toString()); + } + + static String openTypeDescriptor(String getter) { + return getter.startsWith("$k") ? "()Ljava/lang/Object;" : "()" + Translator.TYPE_DESCRIPTOR; + } + + /// The method of a context that answers for a type its code names. + static String getter(Use u) { + return "$d" + mangle(u.key()); + } + + /// The method of a type's context that answers `typeof` of a type + /// argument; of a method's context, of one of the method's own. + static String typeGetter(int index) { + return "$t" + index; + } + + /// A name that is legal in a class file and still says something, with + /// the hash of the whole text after it: sanitizing alone would give + /// `A.B` and `A_B` one name. + private static String mangle(String text) { + String plain = Names.sanitize(text); + int from = Math.max(0, plain.length() - 40); + return plain.substring(from) + "_" + Integer.toHexString(text.hashCode()); + } + + // --------------------------------------------------------------- emission + + /// The class of a generic type's contexts: its static fields as + /// instance fields, and a question for each thing a context has to + /// answer. + void emitContextClass(TypeDef type) { + String name = contextClass(type); + ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_MAXS); + cw.visit(V1_5, ACC_PUBLIC | ACC_ABSTRACT | ACC_SUPER, name, null, Names.OBJECT, null); + for (FieldDef f : type.fields) { + if (isStaticStorage(f)) { + cw.visitField(ACC_PUBLIC, Names.sanitize(f.name), host.names.descriptor(f.type), null, null) + .visitEnd(); + } + } + MethodVisitor mv = cw.visitMethod(ACC_PUBLIC, "", "()V", null, null); + mv.visitCode(); + mv.visitVarInsn(ALOAD, 0); + mv.visitMethodInsn(INVOKESPECIAL, Names.OBJECT, "", "()V", false); + for (FieldDef f : type.fields) { + // A field of struct type always holds an object, as it always + // holds a value in C#. + if (isStaticStorage(f) && host.names.isStruct(f.type)) { + String c = host.names.classRef(f.type); + host.index.requireMethod(c, "", "()V", type.fullName()); + mv.visitVarInsn(ALOAD, 0); + mv.visitTypeInsn(NEW, c); + mv.visitInsn(DUP); + mv.visitMethodInsn(INVOKESPECIAL, c, "", "()V", false); + mv.visitFieldInsn(PUTFIELD, name, Names.sanitize(f.name), host.names.descriptor(f.type)); + } + } + mv.visitInsn(RETURN); + mv.visitMaxs(0, 0); + mv.visitEnd(); + emitQuestions(cw, type.genericParamCount, dependencies(type), openTypes(type)); + cw.visitEnd(); + host.putClass(name, cw.toByteArray()); + } + + void emitContextClass(MethodDef method) { + String name = contextClass(method); + ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_MAXS); + cw.visit(V1_5, ACC_PUBLIC | ACC_ABSTRACT | ACC_SUPER, name, null, Names.OBJECT, null); + MethodVisitor mv = cw.visitMethod(ACC_PUBLIC, "", "()V", null, null); + mv.visitCode(); + mv.visitVarInsn(ALOAD, 0); + mv.visitMethodInsn(INVOKESPECIAL, Names.OBJECT, "", "()V", false); + mv.visitInsn(RETURN); + mv.visitMaxs(0, 0); + mv.visitEnd(); + emitQuestions(cw, method.genericParamCount, dependencies(method), openTypes(method)); + cw.visitEnd(); + host.putClass(name, cw.toByteArray()); + } + + private void emitQuestions(ClassWriter cw, int typeParameters, Map uses, + Map open) { + for (String g : open.keySet()) { + cw.visitMethod(ACC_PUBLIC | ACC_ABSTRACT, g, openTypeDescriptor(g), null, null).visitEnd(); + } + for (int i = 0; i < typeParameters; i++) { + cw.visitMethod(ACC_PUBLIC | ACC_ABSTRACT, typeGetter(i), "()" + Translator.TYPE_DESCRIPTOR, null, null) + .visitEnd(); + } + for (Use u : uses.values()) { + cw.visitMethod(ACC_PUBLIC | ACC_ABSTRACT, getter(u), "()L" + contextClass(u) + ";", null, null) + .visitEnd(); + } + } + + /// The class that holds the one context of a constructed type, or of a + /// constructed generic method, written the first time something names + /// it. Its initializer is where the C# static constructor runs. + String closedClass(Use u, String neededBy) { + if (u.open()) { + throw new TranslationException(neededBy + ": " + u + " still names a type variable"); + } + String base = contextClass(u); + String arguments = u.arguments(); + String name = base + "$" + mangle(arguments); + String known = closedClasses.get(name); + if (known != null) { + if (!known.equals(arguments)) { + throw new TranslationException(neededBy + ": " + u + " and the same type of " + known + + " have one class name once shortened (" + name + ")"); + } + return name; + } + closedClasses.put(name, arguments); + if (depth == 0) { + outermost = u; + } + depth++; + if (depth > DEPTH_LIMIT) { + throw new TranslationException(neededBy + ": " + outermost + " reaches an endless family of" + + " constructed types (" + u.type.fullName() + " names itself with a larger type argument" + + " each time), and one of them has static state; a variable per constructed type cannot be" + + " laid out ahead of time for a family with no end. Keep that state outside the generic" + + " type"); + } + ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_MAXS); + cw.visit(V1_5, ACC_PUBLIC | ACC_FINAL | ACC_SUPER, name, null, base, null); + String descriptor = "L" + base + ";"; + cw.visitField(ACC_PUBLIC | ACC_STATIC | ACC_FINAL, INSTANCE, descriptor, null, null).visitEnd(); + + MethodVisitor mv = cw.visitMethod(ACC_PRIVATE, "", "()V", null, null); + mv.visitCode(); + mv.visitVarInsn(ALOAD, 0); + mv.visitMethodInsn(INVOKESPECIAL, base, "", "()V", false); + mv.visitInsn(RETURN); + mv.visitMaxs(0, 0); + mv.visitEnd(); + + // The context exists before the static constructor runs: that + // constructor is handed it, and whatever it calls that names this + // constructed type reads the field and finds it. + mv = cw.visitMethod(ACC_STATIC, "", "()V", null, null); + mv.visitCode(); + mv.visitTypeInsn(NEW, name); + mv.visitInsn(DUP); + mv.visitMethodInsn(INVOKESPECIAL, name, "", "()V", false); + mv.visitFieldInsn(PUTSTATIC, name, INSTANCE, descriptor); + if (u.method == null && staticConstructor(u.type) != null) { + mv.visitFieldInsn(GETSTATIC, name, INSTANCE, descriptor); + mv.visitMethodInsn(INVOKESTATIC, host.names.className(u.type.fullName()), STATIC_CONSTRUCTOR, + "(" + descriptor + ")V", false); + } + mv.visitInsn(RETURN); + mv.visitMaxs(0, 0); + mv.visitEnd(); + + CilType[] own = u.method == null ? u.typeArgs : u.methodArgs; + for (int i = 0; i < own.length; i++) { + mv = cw.visitMethod(ACC_PUBLIC, typeGetter(i), "()" + Translator.TYPE_DESCRIPTOR, null, null); + mv.visitCode(); + host.emitTypeOf(mv, own[i], neededBy); + mv.visitInsn(ARETURN); + mv.visitMaxs(0, 0); + mv.visitEnd(); + } + Map types = u.method == null ? openTypes(u.type) : openTypes(u.method); + for (Map.Entry e : types.entrySet()) { + CilType closed = e.getValue().substitute(u.typeArgs, u.methodArgs); + mv = cw.visitMethod(ACC_PUBLIC, e.getKey(), openTypeDescriptor(e.getKey()), null, null); + mv.visitCode(); + if (e.getKey().startsWith("$k")) { + mv.visitLdcInsn(closed.toString()); + } else { + host.emitTypeOf(mv, closed, neededBy); + } + mv.visitInsn(ARETURN); + mv.visitMaxs(0, 0); + mv.visitEnd(); + } + Map uses = u.method == null ? dependencies(u.type) : dependencies(u.method); + for (Use open : uses.values()) { + Use closed = open.substitute(u.typeArgs, u.methodArgs); + String d = "L" + contextClass(closed) + ";"; + mv = cw.visitMethod(ACC_PUBLIC, getter(open), "()" + d, null, null); + mv.visitCode(); + mv.visitFieldInsn(GETSTATIC, closedClass(closed, neededBy), INSTANCE, d); + mv.visitInsn(ARETURN); + mv.visitMaxs(0, 0); + mv.visitEnd(); + } + cw.visitEnd(); + host.putClass(name, cw.toByteArray()); + depth--; + return name; + } + + /// What the static constructor of a type with a context is called: it + /// is an ordinary static method that takes the context, run by each + /// constructed type's class and not by the erased class's own + /// initializer, which would run it once for all of them. + static final String STATIC_CONSTRUCTOR = "$cctor"; +} diff --git a/maven/cil-translator/src/main/java/com/codename1/cil/translate/Jvm.java b/maven/cil-translator/src/main/java/com/codename1/cil/translate/Jvm.java new file mode 100644 index 00000000000..81fe9a9f475 --- /dev/null +++ b/maven/cil-translator/src/main/java/com/codename1/cil/translate/Jvm.java @@ -0,0 +1,273 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.cil.translate; + +import com.codename1.cil.metadata.CilType; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; + +/// Opcode arithmetic: which JVM instruction handles a value of a given kind +/// or an array of a given element type. +final class Jvm implements Opcodes { + private Jvm() { + } + + static int loadOp(int kind) { + return kind == Val.REF ? ALOAD : ILOAD + kind; + } + + static int storeOp(int kind) { + return kind == Val.REF ? ASTORE : ISTORE + kind; + } + + static void pushInt(MethodVisitor mv, int value) { + if (value >= -1 && value <= 5) { + mv.visitInsn(ICONST_0 + value); + } else if (value >= Byte.MIN_VALUE && value <= Byte.MAX_VALUE) { + mv.visitIntInsn(BIPUSH, value); + } else if (value >= Short.MIN_VALUE && value <= Short.MAX_VALUE) { + mv.visitIntInsn(SIPUSH, value); + } else { + mv.visitLdcInsn(Integer.valueOf(value)); + } + } + + static void pushDefault(MethodVisitor mv, int kind) { + switch (kind) { + case Val.I4: + mv.visitInsn(ICONST_0); + break; + case Val.I8: + mv.visitInsn(LCONST_0); + break; + case Val.R4: + mv.visitInsn(FCONST_0); + break; + case Val.R8: + mv.visitInsn(DCONST_0); + break; + default: + mv.visitInsn(ACONST_NULL); + break; + } + } + + /// Offset of an element type within the `xALOAD`/`xASTORE` families, + /// which are laid out int, long, float, double, reference, byte, char, + /// short. + private static int arrayOffset(CilType element) { + switch (element.kind) { + case BOOLEAN: + case I1: + case U1: + return 5; + case CHAR: + case U2: + return 6; + case I2: + return 7; + case I4: + case U4: + case I: + case U: + return 0; + case I8: + case U8: + return 1; + case R4: + return 2; + case R8: + return 3; + default: + return 4; + } + } + + static int arrayLoad(CilType element) { + return IALOAD + arrayOffset(element); + } + + static int arrayStore(CilType element) { + return IASTORE + arrayOffset(element); + } + + static int arrayTypeCode(CilType element) { + switch (element.kind) { + case BOOLEAN: + return T_BOOLEAN; + case CHAR: + case U2: + return T_CHAR; + case I1: + case U1: + return T_BYTE; + case I2: + return T_SHORT; + case I8: + case U8: + return T_LONG; + case R4: + return T_FLOAT; + case R8: + return T_DOUBLE; + default: + return T_INT; + } + } + + /// Boxes the primitive on top of the stack as its box class. + /// + /// Through `valueOf`, which hands out one shared object for a small + /// value. That differs from .NET, where every boxing conversion makes a + /// new object: `object a = 1, b = 1;` and then + /// `ReferenceEquals(a, b)` is false there and true here (and `lock` on + /// a box would share a monitor). It is left so on purpose. Boxing is + /// what every `string.Format` argument, every element of a collection + /// of numbers and every `object` parameter goes through, in code that + /// runs each frame, and an allocation for each of those is a cost every + /// script would pay for an identity comparison of two boxes, which no + /// ordinary script makes. + /// `Equals` and `==` on the unboxed values are unaffected. + static void box(MethodVisitor mv, Names names, CilType type) { + CilType p = names.norm(type); + if (p.kind == CilType.Kind.U1) { + // The parameter is a Java byte: 200 has to arrive as -56. + mv.visitInsn(I2B); + } + String c = Names.boxClass(p); + mv.visitMethodInsn(INVOKESTATIC, c, "valueOf", "(" + names.descriptor(p) + ")L" + c + ";", false); + } + + /// Unboxes a value known to be the right box: the result of a generic + /// member, not a cast the C# code asked for. + static void unbox(MethodVisitor mv, Names names, CilType type) { + CilType p = names.norm(type); + String c = Names.boxClass(p); + mv.visitTypeInsn(CHECKCAST, c); + mv.visitMethodInsn(INVOKEVIRTUAL, c, unboxMethod(p), "()" + names.descriptor(p), false); + if (p.kind == CilType.Kind.U1) { + pushInt(mv, 255); + mv.visitInsn(IAND); + } + } + + static String unboxMethod(CilType primitive) { + switch (primitive.kind) { + case BOOLEAN: + return "booleanValue"; + case CHAR: + case U2: + return "charValue"; + case I1: + case U1: + return "byteValue"; + case I2: + return "shortValue"; + case I8: + case U8: + return "longValue"; + case R4: + return "floatValue"; + case R8: + return "doubleValue"; + default: + return "intValue"; + } + } + + /// Suffix of the checked `Interop.unbox...` helper for a primitive, and + /// of the `orDefault...` one. An unsigned type has helpers of its own, + /// because it has a box of its own: see [Names#boxClass]. + static String unboxSuffix(CilType primitive) { + switch (primitive.kind) { + case U1: + return "UByte"; + case U2: + return "UShort"; + case U4: + return "UInt"; + case U8: + return "ULong"; + default: + break; + } + String m = unboxMethod(primitive); + String stem = m.substring(0, m.length() - "Value".length()); + return Character.toUpperCase(stem.charAt(0)) + stem.substring(1); + } + + /// The type an instruction suffix such as the `i4` of `ldelem.i4` names. + static CilType suffixType(String suffix) { + switch (suffix) { + case "i1": + return CilType.I1; + case "u1": + return CilType.U1; + case "i2": + return CilType.I2; + case "u2": + return CilType.U2; + case "i4": + return CilType.I4; + case "u4": + return CilType.U4; + case "i8": + return CilType.I8; + case "i": + return CilType.I; + case "r4": + return CilType.R4; + case "r8": + return CilType.R8; + case "ref": + return CilType.OBJECT; + default: + throw new TranslationException("unknown instruction suffix ." + suffix); + } + } + + /// Size of a primitive in an assembly's static data, or 0 when the type + /// is not one an array literal can hold. + static int byteSize(CilType primitive) { + switch (primitive.kind) { + case BOOLEAN: + case I1: + case U1: + return 1; + case CHAR: + case I2: + case U2: + return 2; + case I4: + case U4: + case R4: + return 4; + case I8: + case U8: + case R8: + return 8; + default: + return 0; + } + } +} diff --git a/maven/cil-translator/src/main/java/com/codename1/cil/translate/MethodTranslator.java b/maven/cil-translator/src/main/java/com/codename1/cil/translate/MethodTranslator.java new file mode 100644 index 00000000000..df11c313be3 --- /dev/null +++ b/maven/cil-translator/src/main/java/com/codename1/cil/translate/MethodTranslator.java @@ -0,0 +1,3258 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.cil.translate; + +import com.codename1.cil.metadata.CilAssembly; +import com.codename1.cil.metadata.CilAssembly.FieldRef; +import com.codename1.cil.metadata.CilAssembly.MethodDef; +import com.codename1.cil.metadata.CilAssembly.MethodRef; +import com.codename1.cil.metadata.CilAssembly.MethodSig; +import com.codename1.cil.metadata.CilAssembly.TypeDef; +import com.codename1.cil.metadata.CilType; +import com.codename1.cil.metadata.MethodBody; +import com.codename1.cil.metadata.MethodBody.Clause; +import com.codename1.cil.metadata.MethodBody.Instr; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.objectweb.asm.Label; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; + +/// Lowers one CIL method body to JVM bytecode. +/// +/// The body is walked once, in IL order, with the type of every evaluation +/// stack entry tracked alongside. One pass is enough because ECMA-335 fixes +/// the stack at an instruction that is only reached by a backward branch to +/// be empty, so the state at every instruction is known by the time it is +/// reached: from the instruction before it, or from a forward branch already +/// seen. +/// +/// What the JVM has no word for is lowered as follows. +/// +/// - **Structs** are objects. A store copies into the object already there +/// (`$assign`) and a value that something else can still see is copied +/// before it is handed on; see [Val#own]. +/// - **Managed references** to a struct are the struct. To anything else they +/// are an array and an index, so a local whose address is taken lives in a +/// one-element array. +/// - **Exceptions**: all the catch clauses of one `try` share a JVM handler +/// that normalises the exception and tests each clause in turn. A `finally` +/// is a catch-all that records how it was entered and resumes from there. +/// - **Checks ParparVM does not make** -- a failed cast, an integer division +/// by zero -- are emitted explicitly. +final class MethodTranslator implements Opcodes { + private static final int PLAIN = 0; + private static final int PROMOTED = 1; + private static final int STRUCT = 2; + private static final int REF_STRUCT = 3; + private static final int REF_PAIR = 4; + + private static final class Var { + CilType type; + int slot; + int mode; + boolean addressTaken; + boolean stored; + } + + private static final class Group { + final List catches = new ArrayList(); + int tryStart; + int tryEnd; + int original; + int wrapped; + final Label dispatch = new Label(); + } + + private static final class Fin { + Clause clause; + int exception; + int resume; + final Label entry = new Label(); + final Label bodyLabel = new Label(); + final List