EthanPisani/unity-builder
1
0
Fork 0
mirror of https://github.com/game-ci/unity-builder.git synced 2025-07-04 12:25:19 -04:00
Code Issues Packages Projects Releases Wiki Activity

Cloud runner develop - latest fixes (#524)

Browse Source 
Cloud runner develop - latest fixes (#524)
This commit is contained in:
Frostebite 2023-03-27 12:14:23 +01:00 committed by GitHub
parent 309d668d63
commit 7abb3a409d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
91 changed files with 2010 additions and 1439 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/build-tests-ubuntu.yml vendored
View File

@ -49,7 +49,7 @@ jobs:
exclude:
- targetPlatform: Android
unityVersion: 2022.2.7f1
cloudRunnerCluster:
providerStrategy:
# - local-docker
- local
projectPath:
@ -109,7 +109,7 @@ jobs:
unityVersion: ${{ matrix.unityVersion }}
targetPlatform: ${{ matrix.targetPlatform }}
customParameters: -profile SomeProfile -someBoolean -someValue exampleValue
cloudRunnerCluster: ${{ matrix.cloudRunnerCluster }}
providerStrategy: ${{ matrix.providerStrategy }}
###########################
# Upload #

19
.github/workflows/cloud-runner-async-checks.yml vendored
View File

@ -23,7 +23,7 @@ env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: eu-west-2
AWS_BASE_STACK_NAME: game-ci-github-pipelines
AWS_STACK_NAME: game-ci-github-pipelines
CLOUD_RUNNER_BRANCH: ${{ github.ref }}
CLOUD_RUNNER_DEBUG: true
CLOUD_RUNNER_DEBUG_TREE: true
@ -39,20 +39,21 @@ jobs:
if: github.event.event_type != 'pull_request_target'
runs-on: ubuntu-latest
steps:
- name: Checkout (default)
uses: actions/checkout@v3
with:
lfs: false
- run: yarn
- run: yarn run cli -m checks-update
timeout-minutes: 180
- timeout-minutes: 180
env:
UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
PROJECT_PATH: test-project
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GIT_PRIVATE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
CLOUD_RUNNER_CLUSTER: local-docker
AWS_BASE_STACK_NAME: game-ci-github-pipelines
AWS_STACK_NAME: game-ci-github-pipelines
CHECKS_UPDATE: ${{ github.event.inputs.checksObject }}
run: |
git clone -b cloud-runner-develop https://github.com/game-ci/unity-builder
cd unity-builder
yarn
ls
yarn run cli -m checks-update

129
.github/workflows/cloud-runner-pipeline.yml → .github/workflows/cloud-runner-ci-pipeline.yml vendored
View File

@ -21,81 +21,140 @@ env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: eu-west-2
AWS_BASE_STACK_NAME: game-ci-team-pipelines
AWS_STACK_NAME: game-ci-team-pipelines
CLOUD_RUNNER_BRANCH: ${{ github.ref }}
CLOUD_RUNNER_DEBUG: true
CLOUD_RUNNER_DEBUG_TREE: true
DEBUG: true
UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
PROJECT_PATH: test-project
UNITY_VERSION: 2019.3.15f1
USE_IL2CPP: false
USE_GKE_GCLOUD_AUTH_PLUGIN: true
GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
jobs:
integrationTests:
name: Integration Tests
smokeTests:
name: Smoke Tests
if: github.event.event_type != 'pull_request_target'
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
cloudRunnerCluster:
- aws
test:
#- 'cloud-runner-async-workflow'
- 'cloud-runner-caching'
# - 'cloud-runner-end2end-caching'
# - 'cloud-runner-end2end-retaining'
- 'cloud-runner-environment'
- 'cloud-runner-hooks'
- 'cloud-runner-local-persistence'
- 'cloud-runner-locking-core'
- 'cloud-runner-locking-get-locked'
providerStrategy:
#- aws
- local-docker
- k8s
#- k8s
steps:
- name: Checkout (default)
uses: actions/checkout@v3
with:
lfs: false
- uses: google-github-actions/auth@v1
with:
credentials_json: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
- name: 'Set up Cloud SDK'
uses: 'google-github-actions/setup-gcloud@v1'
- name: Get GKE cluster credentials
run: |
export USE_GKE_GCLOUD_AUTH_PLUGIN=True
gcloud components install gke-gcloud-auth-plugin
gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: eu-west-2
- uses: google-github-actions/auth@v1
if: matrix.providerStrategy == 'k8s'
with:
credentials_json: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
- name: 'Set up Cloud SDK'
if: matrix.providerStrategy == 'k8s'
uses: 'google-github-actions/setup-gcloud@v1.1.0'
- name: Get GKE cluster credentials
if: matrix.providerStrategy == 'k8s'
run: |
export USE_GKE_GCLOUD_AUTH_PLUGIN=True
gcloud components install gke-gcloud-auth-plugin
gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT
- run: yarn
- run: yarn run test "cloud-runner-async-workflow" --detectOpenHandles --forceExit --runInBand
if: matrix.CloudRunnerCluster != 'local-docker'
timeout-minutes: 180
- run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
timeout-minutes: 35
env:
UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
PROJECT_PATH: test-project
GIT_PRIVATE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
CLOUD_RUNNER_CLUSTER: ${{ matrix.cloudRunnerCluster }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: yarn run test-i --detectOpenHandles --forceExit --runInBand
if: matrix.CloudRunnerCluster == 'local-docker'
timeout-minutes: 180
CLOUD_RUNNER_CLUSTER: ${{ matrix.providerStrategy }}
tests:
# needs:
# - smokeTests
# - buildTargetTests
name: Integration Tests
if: github.event.event_type != 'pull_request_target'
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
providerStrategy:
- aws
- local-docker
- k8s
test:
- 'cloud-runner-async-workflow'
#- 'cloud-runner-caching'
- 'cloud-runner-end2end-locking'
- 'cloud-runner-end2end-caching'
- 'cloud-runner-end2end-retaining'
- 'cloud-runner-environment'
#- 'cloud-runner-hooks'
- 'cloud-runner-s3-steps'
#- 'cloud-runner-local-persistence'
#- 'cloud-runner-locking-core'
#- 'cloud-runner-locking-get-locked'
steps:
- name: Checkout (default)
uses: actions/checkout@v2
with:
lfs: false
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: eu-west-2
- uses: google-github-actions/auth@v1
if: matrix.providerStrategy == 'k8s'
with:
credentials_json: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
- name: 'Set up Cloud SDK'
if: matrix.providerStrategy == 'k8s'
uses: 'google-github-actions/setup-gcloud@v1.1.0'
- name: Get GKE cluster credentials
if: matrix.providerStrategy == 'k8s'
run: |
export USE_GKE_GCLOUD_AUTH_PLUGIN=True
gcloud components install gke-gcloud-auth-plugin
gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT
- run: yarn
- run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
timeout-minutes: 60
env:
UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
PROJECT_PATH: test-project
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TARGET_PLATFORM: StandaloneWindows64
cloudRunnerTests: true
versioning: None
CLOUD_RUNNER_CLUSTER: ${{ matrix.cloudRunnerCluster }}
localBuildTests:
PROVIDER_STRATEGY: ${{ matrix.providerStrategy }}
buildTargetTests:
name: Local Build Target Tests
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
cloudRunnerCluster:
providerStrategy:
#- aws
- local-docker
#- k8s
@ -114,20 +173,18 @@ jobs:
- run: yarn
- uses: ./
id: unity-build
timeout-minutes: 90
timeout-minutes: 30
env:
UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
with:
cloudRunnerTests: true
versioning: None
projectPath: test-project
gitPrivateToken: ${{ secrets.GITHUB_TOKEN }}
targetPlatform: ${{ matrix.targetPlatform }}
cloudRunnerCluster: ${{ matrix.cloudRunnerCluster }}
providerStrategy: ${{ matrix.providerStrategy }}
- run: |
cp ./cloud-runner-cache/cache/${{ steps.unity-build.outputs.CACHE_KEY }}/build/${{ steps.unity-build.outputs.BUILD_ARTIFACT }} ${{ steps.unity-build.outputs.BUILD_ARTIFACT }}
- uses: actions/upload-artifact@v3
with:
name: ${{ matrix.cloudRunnerCluster }} Build (${{ matrix.targetPlatform }})
name: ${{ matrix.providerStrategy }} Build (${{ matrix.targetPlatform }})
path: ${{ steps.unity-build.outputs.BUILD_ARTIFACT }}
retention-days: 14

8
action.yml
View File

@ -118,7 +118,7 @@ inputs:
description:
'[CloudRunner] Run a pre build job after the repository setup but before the build job (in yaml format with the
keys image, secrets (name, value object array), command line string)'
customStepFiles:
containerHookFiles:
required: false
default: ''
description:
@ -130,7 +130,7 @@ inputs:
description:
'[CloudRunner] Specify the names (by file name) of custom hooks to run before or after cloud runner jobs, must
match a yaml step file inside your repo in the folder .game-ci/hooks/'
customJobHooks:
customCommandHooks:
required: false
default: ''
description: '[CloudRunner] Specify custom commands and trigger hooks (injects commands into jobs)'
@ -140,11 +140,11 @@ inputs:
description:
'[CloudRunner] Run a custom job instead of the standard build automation for cloud runner (in yaml format with the
keys image, secrets (name, value object array), command line string)'
awsBaseStackName:
awsStackName:
default: 'game-ci'
required: false
description: '[CloudRunner] The Cloud Formation stack name that must be setup before using this option.'
cloudRunnerCluster:
providerStrategy:
default: 'local'
required: false
description:

BIN
dist/index.js generated vendored
View File

Binary file not shown.

BIN
dist/index.js.map generated vendored
View File

Binary file not shown.

2
game-ci/hooks/my-test-hook-post-build.yaml → game-ci/command-hooks/my-test-hook-post-build.yaml
View File

@ -1,3 +1,3 @@
hook: after-build
hook: after
commands: |
echo "after-build hook test!"

2
game-ci/hooks/my-test-hook-pre-build.yaml → game-ci/command-hooks/my-test-hook-pre-build.yaml
View File

@ -1,3 +1,3 @@
hook: before-build
hook: before
commands: |
echo "before-build hook test!!"

0
game-ci/steps/my-test-step-post-build.yaml → game-ci/container-hooks/my-test-step-post-build.yaml
View File

0
game-ci/steps/my-test-step-pre-build.yaml → game-ci/container-hooks/my-test-step-pre-build.yaml
View File

12
package.json
View File

@ -12,17 +12,17 @@
"lint": "prettier --check \"src/**/*.{js,ts}\" && eslint src/**/*.ts",
"format": "prettier --write \"src/**/*.{js,ts}\"",
"cli": "yarn ts-node src/index.ts -m cli",
"gcp-secrets-tests": "cross-env cloudRunnerCluster=aws cloudRunnerTests=true readInputOverrideCommand=\"gcp-secret-manager\" populateOverride=true readInputFromOverrideList=UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD yarn test -i -t \"cloud runner\"",
"gcp-secrets-tests": "cross-env providerStrategy=aws cloudRunnerTests=true readInputOverrideCommand=\"gcp-secret-manager\" populateOverride=true readInputFromOverrideList=UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD yarn test -i -t \"cloud runner\"",
"gcp-secrets-cli": "cross-env cloudRunnerTests=true readInputOverrideCommand=\"gcp-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --readInputFromOverrideList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD",
"aws-secrets-cli": "cross-env cloudRunnerTests=true readInputOverrideCommand=\"aws-secret-manager\" yarn ts-node src/index.ts -m cli --populateOverride true --readInputFromOverrideList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD",
"cli-aws": "cross-env cloudRunnerCluster=aws yarn run test-cli",
"cli-k8s": "cross-env cloudRunnerCluster=k8s yarn run test-cli",
"cli-aws": "cross-env providerStrategy=aws yarn run test-cli",
"cli-k8s": "cross-env providerStrategy=k8s yarn run test-cli",
"test-cli": "cross-env cloudRunnerTests=true yarn ts-node src/index.ts -m cli --projectPath test-project",
"test": "jest",
"test-i": "cross-env cloudRunnerTests=true yarn test -i -t \"cloud runner\"",
"test-i-*": "yarn run test-i-aws && yarn run test-i-k8s",
"test-i-aws": "cross-env cloudRunnerTests=true cloudRunnerCluster=aws yarn test -i -t \"cloud runner\"",
"test-i-k8s": "cross-env cloudRunnerTests=true cloudRunnerCluster=k8s yarn test -i -t \"cloud runner\""
"test-i-aws": "cross-env cloudRunnerTests=true providerStrategy=aws yarn test -i -t \"cloud runner\"",
"test-i-k8s": "cross-env cloudRunnerTests=true providerStrategy=k8s yarn test -i -t \"cloud runner\""
},
"engines": {
"node": ">=16.x"
@ -44,7 +44,7 @@
"reflect-metadata": "^0.1.13",
"semver": "^7.3.5",
"unity-changeset": "^2.0.0",
"uuid": "^8.3.2",
"uuid": "^9.0.0",
"yaml": "^1.10.2"
},
"devDependencies": {

2
src/index.ts
View File

@ -19,7 +19,7 @@ async function runMain() {
const buildParameters = await BuildParameters.create();
const baseImage = new ImageTag(buildParameters);
if (buildParameters.cloudRunnerCluster === 'local') {
if (buildParameters.providerStrategy === 'local') {
core.info('Building locally');
await PlatformSetup.setup(buildParameters, actionFolder);
if (process.platform === 'darwin') {

78
src/model/build-parameters.ts
View File

@ -1,7 +1,7 @@
import { customAlphabet } from 'nanoid';
import AndroidVersioning from './android-versioning';
import CloudRunnerConstants from './cloud-runner/services/cloud-runner-constants';
import CloudRunnerBuildGuid from './cloud-runner/services/cloud-runner-guid';
import CloudRunnerConstants from './cloud-runner/options/cloud-runner-constants';
import CloudRunnerBuildGuid from './cloud-runner/options/cloud-runner-guid';
import Input from './input';
import Platform from './platform';
import UnityVersioning from './unity-versioning';
@ -10,7 +10,8 @@ import { GitRepoReader } from './input-readers/git-repo';
import { GithubCliReader } from './input-readers/github-cli';
import { Cli } from './cli/cli';
import GitHub from './github';
import CloudRunnerOptions from './cloud-runner/cloud-runner-options';
import CloudRunnerOptions from './cloud-runner/options/cloud-runner-options';
import CloudRunner from './cloud-runner/cloud-runner';
class BuildParameters {
// eslint-disable-next-line no-undef
@ -41,24 +42,23 @@ class BuildParameters {
public customParameters!: string;
public sshAgent!: string;
public cloudRunnerCluster!: string;
public awsBaseStackName!: string;
public providerStrategy!: string;
public gitPrivateToken!: string;
public awsStackName!: string;
public kubeConfig!: string;
public cloudRunnerMemory!: string | undefined;
public cloudRunnerCpu!: string | undefined;
public containerMemory!: string;
public containerCpu!: string;
public kubeVolumeSize!: string;
public kubeVolume!: string;
public kubeStorageClass!: string;
public chownFilesTo!: string;
public customJobHooks!: string;
public readInputFromOverrideList!: string;
public readInputOverrideCommand!: string;
public commandHooks!: string;
public pullInputList!: string[];
public inputPullCommand!: string;
public cacheKey!: string;
public postBuildSteps!: string;
public preBuildSteps!: string;
public postBuildContainerHooks!: string;
public preBuildContainerHooks!: string;
public customJob!: string;
public runNumber!: string;
public branch!: string;
@ -68,17 +68,23 @@ class BuildParameters {
public buildGuid!: string;
public cloudRunnerBranch!: string;
public cloudRunnerDebug!: boolean | undefined;
public cloudRunnerBuilderPlatform!: string | undefined;
public buildPlatform!: string | undefined;
public isCliMode!: boolean;
public retainWorkspace!: boolean;
public maxRetainedWorkspaces!: number;
public useSharedLargePackages!: boolean;
public useLz4Compression!: boolean;
public garbageCollectionMaxAge!: number;
public constantGarbageCollection!: boolean;
public useLargePackages!: boolean;
public useCompressionStrategy!: boolean;
public garbageMaxAge!: number;
public githubChecks!: boolean;
public asyncWorkflow!: boolean;
public githubCheckId!: string;
public finalHooks!: string[];
public skipLfs!: boolean;
public skipCache!: boolean;
public cacheUnityInstallationOnMac!: boolean;
public unityHubVersionOnMac!: string;
public static shouldUseRetainedWorkspaceMode(buildParameters: BuildParameters) {
return buildParameters.maxRetainedWorkspaces > 0 && CloudRunner.lockedWorkspace !== ``;
}
static async create(): Promise<BuildParameters> {
const buildFile = this.parseBuildFile(Input.buildName, Input.targetPlatform, Input.androidExportType);
@ -144,16 +150,15 @@ class BuildParameters {
sshAgent: Input.sshAgent,
gitPrivateToken: Input.gitPrivateToken || (await GithubCliReader.GetGitHubAuthToken()),
chownFilesTo: Input.chownFilesTo,
cloudRunnerCluster: CloudRunnerOptions.cloudRunnerCluster,
cloudRunnerBuilderPlatform: CloudRunnerOptions.cloudRunnerBuilderPlatform,
awsBaseStackName: CloudRunnerOptions.awsBaseStackName,
providerStrategy: CloudRunnerOptions.providerStrategy,
buildPlatform: CloudRunnerOptions.buildPlatform,
kubeConfig: CloudRunnerOptions.kubeConfig,
cloudRunnerMemory: CloudRunnerOptions.cloudRunnerMemory,
cloudRunnerCpu: CloudRunnerOptions.cloudRunnerCpu,
containerMemory: CloudRunnerOptions.containerMemory,
containerCpu: CloudRunnerOptions.containerCpu,
kubeVolumeSize: CloudRunnerOptions.kubeVolumeSize,
kubeVolume: CloudRunnerOptions.kubeVolume,
postBuildSteps: CloudRunnerOptions.postBuildSteps,
preBuildSteps: CloudRunnerOptions.preBuildSteps,
postBuildContainerHooks: CloudRunnerOptions.postBuildContainerHooks,
preBuildContainerHooks: CloudRunnerOptions.preBuildContainerHooks,
customJob: CloudRunnerOptions.customJob,
runNumber: Input.runNumber,
branch: Input.branch.replace('/head', '') || (await GitRepoReader.GetBranch()),
@ -161,22 +166,25 @@ class BuildParameters {
cloudRunnerDebug: CloudRunnerOptions.cloudRunnerDebug,
githubRepo: Input.githubRepo || (await GitRepoReader.GetRemote()) || 'game-ci/unity-builder',
isCliMode: Cli.isCliMode,
awsStackName: CloudRunnerOptions.awsBaseStackName,
awsStackName: CloudRunnerOptions.awsStackName,
gitSha: Input.gitSha,
logId: customAlphabet(CloudRunnerConstants.alphabet, 9)(),
buildGuid: CloudRunnerBuildGuid.generateGuid(Input.runNumber, Input.targetPlatform),
customJobHooks: CloudRunnerOptions.customJobHooks(),
readInputOverrideCommand: CloudRunnerOptions.readInputOverrideCommand(),
readInputFromOverrideList: CloudRunnerOptions.readInputFromOverrideList(),
commandHooks: CloudRunnerOptions.commandHooks,
inputPullCommand: CloudRunnerOptions.inputPullCommand,
pullInputList: CloudRunnerOptions.pullInputList,
kubeStorageClass: CloudRunnerOptions.kubeStorageClass,
cacheKey: CloudRunnerOptions.cacheKey,
retainWorkspace: CloudRunnerOptions.retainWorkspaces,
useSharedLargePackages: CloudRunnerOptions.useSharedLargePackages,
useLz4Compression: CloudRunnerOptions.useLz4Compression,
maxRetainedWorkspaces: CloudRunnerOptions.maxRetainedWorkspaces,
constantGarbageCollection: CloudRunnerOptions.constantGarbageCollection,
garbageCollectionMaxAge: CloudRunnerOptions.garbageCollectionMaxAge,
maxRetainedWorkspaces: Number.parseInt(CloudRunnerOptions.maxRetainedWorkspaces),
useLargePackages: CloudRunnerOptions.useLargePackages,
useCompressionStrategy: CloudRunnerOptions.useCompressionStrategy,
garbageMaxAge: CloudRunnerOptions.garbageMaxAge,
githubChecks: CloudRunnerOptions.githubChecks,
asyncWorkflow: CloudRunnerOptions.asyncCloudRunner,
githubCheckId: CloudRunnerOptions.githubCheckId,
finalHooks: CloudRunnerOptions.finalHooks,
skipLfs: CloudRunnerOptions.skipLfs,
skipCache: CloudRunnerOptions.skipCache,
cacheUnityInstallationOnMac: Input.cacheUnityInstallationOnMac,
unityHubVersionOnMac: Input.unityHubVersionOnMac,
};

44
src/model/cli/cli.ts
View File

@ -2,17 +2,16 @@ import { Command } from 'commander-ts';
import { BuildParameters, CloudRunner, ImageTag, Input } from '..';
import * as core from '@actions/core';
import { ActionYamlReader } from '../input-readers/action-yaml';
import CloudRunnerLogger from '../cloud-runner/services/cloud-runner-logger';
import CloudRunnerQueryOverride from '../cloud-runner/services/cloud-runner-query-override';
import CloudRunnerLogger from '../cloud-runner/services/core/cloud-runner-logger';
import CloudRunnerQueryOverride from '../cloud-runner/options/cloud-runner-query-override';
import { CliFunction, CliFunctionsRepository } from './cli-functions-repository';
import { Caching } from '../cloud-runner/remote-client/caching';
import { LfsHashing } from '../cloud-runner/services/lfs-hashing';
import { LfsHashing } from '../cloud-runner/services/utility/lfs-hashing';
import { RemoteClient } from '../cloud-runner/remote-client';
import CloudRunnerOptionsReader from '../cloud-runner/services/cloud-runner-options-reader';
import CloudRunnerOptionsReader from '../cloud-runner/options/cloud-runner-options-reader';
import GitHub from '../github';
import { TaskParameterSerializer } from '../cloud-runner/services/task-parameter-serializer';
import { CloudRunnerFolders } from '../cloud-runner/services/cloud-runner-folders';
import { CloudRunnerSystem } from '../cloud-runner/services/cloud-runner-system';
import { CloudRunnerFolders } from '../cloud-runner/options/cloud-runner-folders';
import { CloudRunnerSystem } from '../cloud-runner/services/core/cloud-runner-system';
import { OptionValues } from 'commander';
import { InputKey } from '../input';
@ -73,12 +72,14 @@ export class Cli {
CloudRunnerLogger.log(`Entrypoint: ${results.key}`);
Cli.options!.versioning = 'None';
const buildParameter = TaskParameterSerializer.readBuildParameterFromEnvironment();
CloudRunner.buildParameters = await BuildParameters.create();
CloudRunner.buildParameters.buildGuid = process.env.BUILD_GUID || ``;
CloudRunnerLogger.log(`Build Params:
${JSON.stringify(buildParameter, undefined, 4)}
${JSON.stringify(CloudRunner.buildParameters, undefined, 4)}
`);
CloudRunner.buildParameters = buildParameter;
CloudRunner.lockedWorkspace = process.env.LOCKED_WORKSPACE;
CloudRunner.lockedWorkspace = process.env.LOCKED_WORKSPACE || ``;
CloudRunnerLogger.log(`Locked Workspace: ${CloudRunner.lockedWorkspace}`);
await CloudRunner.setup(CloudRunner.buildParameters);
return await results.target[results.propertyKey](Cli.options);
}
@ -116,12 +117,16 @@ export class Cli {
public static async asyncronousWorkflow(): Promise<string> {
const buildParameter = await BuildParameters.create();
const baseImage = new ImageTag(buildParameter);
await CloudRunner.setup(buildParameter);
return await CloudRunner.run(buildParameter, baseImage.toString());
}
@CliFunction(`checks-update`, `runs a cloud runner build`)
public static async checksUpdate() {
const buildParameter = await BuildParameters.create();
await CloudRunner.setup(buildParameter);
const input = JSON.parse(process.env.CHECKS_UPDATE || ``);
core.info(`Checks Update ${process.env.CHECKS_UPDATE}`);
if (input.mode === `create`) {
@ -185,7 +190,7 @@ export class Cli {
`build-${CloudRunner.buildParameters.buildGuid}`,
);
if (!CloudRunner.buildParameters.retainWorkspace) {
if (!BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)) {
await CloudRunnerSystem.Run(
`rm -r ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
);
@ -193,21 +198,6 @@ export class Cli {
await RemoteClient.runCustomHookFiles(`after-build`);
const parameters = await BuildParameters.create();
CloudRunner.setup(parameters);
if (parameters.constantGarbageCollection) {
await CloudRunnerSystem.Run(
`find /${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.buildVolumeFolder)}/ -name '*.*' -mmin +${
parameters.garbageCollectionMaxAge * 60
} -delete`,
);
await CloudRunnerSystem.Run(
`find ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderForAllFull)} -name '*.*' -mmin +${
parameters.garbageCollectionMaxAge * 60
} -delete`,
);
}
return new Promise((result) => result(``));
}
}

107
src/model/cloud-runner/cloud-runner.ts
View File

@ -1,34 +1,42 @@
import AwsBuildPlatform from './providers/aws';
import { BuildParameters, Input } from '..';
import Kubernetes from './providers/k8s';
import CloudRunnerLogger from './services/cloud-runner-logger';
import { CloudRunnerStepState } from './cloud-runner-step-state';
import CloudRunnerLogger from './services/core/cloud-runner-logger';
import { CloudRunnerStepParameters } from './options/cloud-runner-step-parameters';
import { WorkflowCompositionRoot } from './workflows/workflow-composition-root';
import { CloudRunnerError } from './error/cloud-runner-error';
import { TaskParameterSerializer } from './services/task-parameter-serializer';
import { TaskParameterSerializer } from './services/core/task-parameter-serializer';
import * as core from '@actions/core';
import CloudRunnerSecret from './services/cloud-runner-secret';
import CloudRunnerSecret from './options/cloud-runner-secret';
import { ProviderInterface } from './providers/provider-interface';
import CloudRunnerEnvironmentVariable from './services/cloud-runner-environment-variable';
import CloudRunnerEnvironmentVariable from './options/cloud-runner-environment-variable';
import TestCloudRunner from './providers/test';
import LocalCloudRunner from './providers/local';
import LocalDockerCloudRunner from './providers/docker';
import GitHub from '../github';
import SharedWorkspaceLocking from './services/shared-workspace-locking';
import SharedWorkspaceLocking from './services/core/shared-workspace-locking';
import { FollowLogStreamService } from './services/core/follow-log-stream-service';
class CloudRunner {
public static Provider: ProviderInterface;
public static buildParameters: BuildParameters;
private static defaultSecrets: CloudRunnerSecret[];
private static cloudRunnerEnvironmentVariables: CloudRunnerEnvironmentVariable[];
static lockedWorkspace: string | undefined;
static lockedWorkspace: string = ``;
public static readonly retainedWorkspacePrefix: string = `retained-workspace`;
public static githubCheckId: number | string;
public static setup(buildParameters: BuildParameters) {
public static get isCloudRunnerEnvironment() {
return process.env[`GITHUB_ACTIONS`] !== `true`;
}
public static get isCloudRunnerAsyncEnvironment() {
return process.env[`ASYNC_WORKFLOW`] === `true`;
}
public static async setup(buildParameters: BuildParameters) {
CloudRunnerLogger.setup();
CloudRunnerLogger.log(`Setting up cloud runner`);
CloudRunner.buildParameters = buildParameters;
if (CloudRunner.buildParameters.githubCheckId === ``) {
CloudRunner.buildParameters.githubCheckId = await GitHub.createGitHubCheck(CloudRunner.buildParameters.buildGuid);
}
CloudRunner.setupSelectedBuildPlatform();
CloudRunner.defaultSecrets = TaskParameterSerializer.readDefaultSecrets();
CloudRunner.cloudRunnerEnvironmentVariables =
@ -46,15 +54,16 @@ class CloudRunner {
core.setOutput(
Input.ToEnvVarFormat(`buildArtifact`),
`build-${CloudRunner.buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
}`,
);
}
FollowLogStreamService.Reset();
}
private static setupSelectedBuildPlatform() {
CloudRunnerLogger.log(`Cloud Runner platform selected ${CloudRunner.buildParameters.cloudRunnerCluster}`);
switch (CloudRunner.buildParameters.cloudRunnerCluster) {
CloudRunnerLogger.log(`Cloud Runner platform selected ${CloudRunner.buildParameters.providerStrategy}`);
switch (CloudRunner.buildParameters.providerStrategy) {
case 'k8s':
CloudRunner.Provider = new Kubernetes(CloudRunner.buildParameters);
break;
@ -74,14 +83,20 @@ class CloudRunner {
}
static async run(buildParameters: BuildParameters, baseImage: string) {
CloudRunner.setup(buildParameters);
await CloudRunner.setup(buildParameters);
if (!CloudRunner.buildParameters.isCliMode) core.startGroup('Setup shared cloud runner resources');
await CloudRunner.Provider.setupWorkflow(
CloudRunner.buildParameters.buildGuid,
CloudRunner.buildParameters,
CloudRunner.buildParameters.branch,
CloudRunner.defaultSecrets,
);
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
try {
CloudRunner.githubCheckId = await GitHub.createGitHubCheck(CloudRunner.buildParameters.buildGuid);
if (buildParameters.maxRetainedWorkspaces > 0) {
CloudRunner.lockedWorkspace = SharedWorkspaceLocking.NewWorkspaceName();
if (buildParameters.retainWorkspace) {
CloudRunner.lockedWorkspace = `${CloudRunner.retainedWorkspacePrefix}-${CloudRunner.buildParameters.buildGuid}`;
const result = await SharedWorkspaceLocking.GetOrCreateLockedWorkspace(
const result = await SharedWorkspaceLocking.GetLockedWorkspace(
CloudRunner.lockedWorkspace,
CloudRunner.buildParameters.buildGuid,
CloudRunner.buildParameters,
@ -95,21 +110,21 @@ class CloudRunner {
];
} else {
CloudRunnerLogger.log(`Max retained workspaces reached ${buildParameters.maxRetainedWorkspaces}`);
buildParameters.retainWorkspace = false;
CloudRunner.lockedWorkspace = undefined;
buildParameters.maxRetainedWorkspaces = 0;
CloudRunner.lockedWorkspace = ``;
}
}
if (!CloudRunner.buildParameters.isCliMode) core.startGroup('Setup shared cloud runner resources');
await CloudRunner.Provider.setupWorkflow(
CloudRunner.buildParameters.buildGuid,
CloudRunner.buildParameters,
CloudRunner.buildParameters.branch,
CloudRunner.defaultSecrets,
);
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
await GitHub.updateGitHubCheck(CloudRunner.buildParameters.buildGuid, CloudRunner.buildParameters.buildGuid);
const content = { ...CloudRunner.buildParameters };
content.gitPrivateToken = ``;
content.unitySerial = ``;
const jsonContent = JSON.stringify(content, undefined, 4);
await GitHub.updateGitHubCheck(jsonContent, CloudRunner.buildParameters.buildGuid);
const output = await new WorkflowCompositionRoot().run(
new CloudRunnerStepState(baseImage, CloudRunner.cloudRunnerEnvironmentVariables, CloudRunner.defaultSecrets),
new CloudRunnerStepParameters(
baseImage,
CloudRunner.cloudRunnerEnvironmentVariables,
CloudRunner.defaultSecrets,
),
);
if (!CloudRunner.buildParameters.isCliMode) core.startGroup('Cleanup shared cloud runner resources');
await CloudRunner.Provider.cleanupWorkflow(
@ -122,22 +137,40 @@ class CloudRunner {
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
await GitHub.updateGitHubCheck(CloudRunner.buildParameters.buildGuid, `success`, `success`, `completed`);
if (CloudRunner.buildParameters.retainWorkspace) {
if (BuildParameters.shouldUseRetainedWorkspaceMode(buildParameters)) {
const workspace = CloudRunner.lockedWorkspace || ``;
await SharedWorkspaceLocking.ReleaseWorkspace(
CloudRunner.lockedWorkspace || ``,
workspace,
CloudRunner.buildParameters.buildGuid,
CloudRunner.buildParameters,
);
CloudRunner.lockedWorkspace = undefined;
const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(workspace, CloudRunner.buildParameters);
if (isLocked) {
throw new Error(
`still locked after releasing ${await SharedWorkspaceLocking.GetAllLocksForWorkspace(
workspace,
buildParameters,
)}`,
);
}
CloudRunner.lockedWorkspace = ``;
}
await GitHub.triggerWorkflowOnComplete(CloudRunner.buildParameters.finalHooks);
if (buildParameters.constantGarbageCollection) {
CloudRunner.Provider.garbageCollect(``, true, buildParameters.garbageCollectionMaxAge, true, true);
CloudRunner.Provider.garbageCollect(``, true, buildParameters.garbageMaxAge, true, true);
}
return output;
} catch (error) {
await GitHub.updateGitHubCheck(CloudRunner.buildParameters.buildGuid, error, `failure`, `completed`);
} catch (error: any) {
CloudRunnerLogger.log(JSON.stringify(error, undefined, 4));
await GitHub.updateGitHubCheck(
CloudRunner.buildParameters.buildGuid,
`Failed - Error ${error?.message || error}`,
`failure`,
`completed`,
);
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
await CloudRunnerError.handleException(error, CloudRunner.buildParameters, CloudRunner.defaultSecrets);
throw error;

4
src/model/cloud-runner/error/cloud-runner-error.ts
View File

@ -1,7 +1,7 @@
import CloudRunnerLogger from '../services/cloud-runner-logger';
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
import * as core from '@actions/core';
import CloudRunner from '../cloud-runner';
import CloudRunnerSecret from '../services/cloud-runner-secret';
import CloudRunnerSecret from '../options/cloud-runner-secret';
import BuildParameters from '../../build-parameters';
export class CloudRunnerError {

0
src/model/cloud-runner/services/cloud-runner-constants.ts → src/model/cloud-runner/options/cloud-runner-constants.ts
View File

0
src/model/cloud-runner/services/cloud-runner-environment-variable.ts → src/model/cloud-runner/options/cloud-runner-environment-variable.ts
View File

7
src/model/cloud-runner/services/cloud-runner-folders.ts → src/model/cloud-runner/options/cloud-runner-folders.ts
View File

@ -1,6 +1,7 @@
import path from 'node:path';
import CloudRunnerOptions from '../cloud-runner-options';
import CloudRunner from './../cloud-runner';
import CloudRunnerOptions from './cloud-runner-options';
import CloudRunner from '../cloud-runner';
import BuildParameters from '../../build-parameters';
export class CloudRunnerFolders {
public static readonly repositoryFolder = 'repo';
@ -12,7 +13,7 @@ export class CloudRunnerFolders {
// Only the following paths that do not start a path.join with another "Full" suffixed property need to start with an absolute /
public static get uniqueCloudRunnerJobFolderAbsolute(): string {
return CloudRunner.buildParameters && CloudRunner.buildParameters.retainWorkspace && CloudRunner.lockedWorkspace
return CloudRunner.buildParameters && BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)
? path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.lockedWorkspace)
: path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.buildParameters.buildGuid);
}

0
src/model/cloud-runner/services/cloud-runner-guid.ts → src/model/cloud-runner/options/cloud-runner-guid.ts
View File

2
src/model/cloud-runner/services/cloud-runner-options-reader.ts → src/model/cloud-runner/options/cloud-runner-options-reader.ts
View File

@ -1,5 +1,5 @@
import Input from '../../input';
import CloudRunnerOptions from '../cloud-runner-options';
import CloudRunnerOptions from './cloud-runner-options';
class CloudRunnerOptionsReader {
static GetProperties() {

142
src/model/cloud-runner/cloud-runner-options.ts → src/model/cloud-runner/options/cloud-runner-options.ts
View File

@ -1,6 +1,6 @@
import { Cli } from '../cli/cli';
import CloudRunnerQueryOverride from './services/cloud-runner-query-override';
import GitHub from '../github';
import { Cli } from '../../cli/cli';
import CloudRunnerQueryOverride from './cloud-runner-query-override';
import GitHub from '../../github';
import * as core from '@actions/core';
class CloudRunnerOptions {
@ -58,7 +58,12 @@ class CloudRunnerOptions {
// GitHub parameters
// ### ### ###
static get githubChecks(): boolean {
return CloudRunnerOptions.getInput('githubChecks') === 'true' || false;
const value = CloudRunnerOptions.getInput('githubChecks');
return value === `true` || false;
}
static get githubCheckId(): string {
return CloudRunnerOptions.getInput('githubCheckId') || ``;
}
static get githubOwner(): string {
@ -69,6 +74,10 @@ class CloudRunnerOptions {
return CloudRunnerOptions.getInput('githubRepoName') || CloudRunnerOptions.githubRepo?.split(`/`)[1] || '';
}
static get finalHooks(): string[] {
return CloudRunnerOptions.getInput('finalHooks')?.split(',') || [];
}
// ### ### ###
// Git syncronization parameters
// ### ### ###
@ -76,59 +85,54 @@ class CloudRunnerOptions {
static get githubRepo(): string | undefined {
return CloudRunnerOptions.getInput('GITHUB_REPOSITORY') || CloudRunnerOptions.getInput('GITHUB_REPO') || undefined;
}
static get branch(): string {
if (CloudRunnerOptions.getInput(`GITHUB_REF`)) {
return CloudRunnerOptions.getInput(`GITHUB_REF`)!.replace('refs/', '').replace(`head/`, '').replace(`heads/`, '');
return (
CloudRunnerOptions.getInput(`GITHUB_REF`)?.replace('refs/', '').replace(`head/`, '').replace(`heads/`, '') || ``
);
} else if (CloudRunnerOptions.getInput('branch')) {
return CloudRunnerOptions.getInput('branch')!;
return CloudRunnerOptions.getInput('branch') || ``;
} else {
return '';
}
}
static get gitSha(): string | undefined {
if (CloudRunnerOptions.getInput(`GITHUB_SHA`)) {
return CloudRunnerOptions.getInput(`GITHUB_SHA`)!;
} else if (CloudRunnerOptions.getInput(`GitSHA`)) {
return CloudRunnerOptions.getInput(`GitSHA`)!;
}
}
// ### ### ###
// Cloud Runner parameters
// ### ### ###
static get cloudRunnerBuilderPlatform(): string | undefined {
const input = CloudRunnerOptions.getInput('cloudRunnerBuilderPlatform');
static get buildPlatform(): string {
const input = CloudRunnerOptions.getInput('buildPlatform');
if (input) {
return input;
}
if (CloudRunnerOptions.cloudRunnerCluster !== 'local') {
if (CloudRunnerOptions.providerStrategy !== 'local') {
return 'linux';
}
return;
return ``;
}
static get cloudRunnerBranch(): string {
return CloudRunnerOptions.getInput('cloudRunnerBranch') || 'main';
}
static get cloudRunnerCluster(): string {
static get providerStrategy(): string {
const provider =
CloudRunnerOptions.getInput('cloudRunnerCluster') || CloudRunnerOptions.getInput('providerStrategy');
if (Cli.isCliMode) {
return CloudRunnerOptions.getInput('cloudRunnerCluster') || 'aws';
return provider || 'aws';
}
return CloudRunnerOptions.getInput('cloudRunnerCluster') || 'local';
return provider || 'local';
}
static get cloudRunnerCpu(): string | undefined {
return CloudRunnerOptions.getInput('cloudRunnerCpu');
static get containerCpu(): string {
return CloudRunnerOptions.getInput('containerCpu') || `1024`;
}
static get cloudRunnerMemory(): string | undefined {
return CloudRunnerOptions.getInput('cloudRunnerMemory');
static get containerMemory(): string {
return CloudRunnerOptions.getInput('containerMemory') || `3072`;
}
static get customJob(): string {
@ -139,40 +143,40 @@ class CloudRunnerOptions {
// Custom commands from files parameters
// ### ### ###
static get customStepFiles(): string[] {
return CloudRunnerOptions.getInput('customStepFiles')?.split(`,`) || [];
static get containerHookFiles(): string[] {
return CloudRunnerOptions.getInput('containerHookFiles')?.split(`,`) || [];
}
static get customHookFiles(): string[] {
return CloudRunnerOptions.getInput('customHookFiles')?.split(`,`) || [];
static get commandHookFiles(): string[] {
return CloudRunnerOptions.getInput('commandHookFiles')?.split(`,`) || [];
}
// ### ### ###
// Custom commands from yaml parameters
// ### ### ###
static customJobHooks(): string {
return CloudRunnerOptions.getInput('customJobHooks') || '';
static get commandHooks(): string {
return CloudRunnerOptions.getInput('commandHooks') || '';
}
static get postBuildSteps(): string {
return CloudRunnerOptions.getInput('postBuildSteps') || '';
static get postBuildContainerHooks(): string {
return CloudRunnerOptions.getInput('postBuildContainerHooks') || '';
}
static get preBuildSteps(): string {
return CloudRunnerOptions.getInput('preBuildSteps') || '';
static get preBuildContainerHooks(): string {
return CloudRunnerOptions.getInput('preBuildContainerHooks') || '';
}
// ### ### ###
// Input override handling
// ### ### ###
static readInputFromOverrideList(): string {
return CloudRunnerOptions.getInput('readInputFromOverrideList') || '';
static get pullInputList(): string[] {
return CloudRunnerOptions.getInput('pullInputList')?.split(`,`) || [];
}
static readInputOverrideCommand(): string {
const value = CloudRunnerOptions.getInput('readInputOverrideCommand');
static get inputPullCommand(): string {
const value = CloudRunnerOptions.getInput('inputPullCommand');
if (value === 'gcp-secret-manager') {
return 'gcloud secrets versions access 1 --secret="{0}"';
@ -187,8 +191,8 @@ class CloudRunnerOptions {
// Aws
// ### ### ###
static get awsBaseStackName(): string {
return CloudRunnerOptions.getInput('awsBaseStackName') || 'game-ci';
static get awsStackName() {
return CloudRunnerOptions.getInput('awsStackName') || 'game-ci';
}
// ### ### ###
@ -204,7 +208,7 @@ class CloudRunnerOptions {
}
static get kubeVolumeSize(): string {
return CloudRunnerOptions.getInput('kubeVolumeSize') || '5Gi';
return CloudRunnerOptions.getInput('kubeVolumeSize') || '25Gi';
}
static get kubeStorageClass(): string {
@ -225,40 +229,34 @@ class CloudRunnerOptions {
static get cloudRunnerDebug(): boolean {
return (
CloudRunnerOptions.getInput(`cloudRunnerTests`) === 'true' ||
CloudRunnerOptions.getInput(`cloudRunnerDebug`) === 'true' ||
CloudRunnerOptions.getInput(`cloudRunnerTests`) === `true` ||
CloudRunnerOptions.getInput(`cloudRunnerDebug`) === `true` ||
CloudRunnerOptions.getInput(`cloudRunnerDebugTree`) === `true` ||
CloudRunnerOptions.getInput(`cloudRunnerDebugEnv`) === `true` ||
false
);
}
static get cloudRunnerDebugTree(): string | boolean {
return CloudRunnerOptions.getInput(`cloudRunnerDebugTree`) || false;
static get skipLfs(): boolean {
return CloudRunnerOptions.getInput(`skipLfs`) === `true`;
}
static get cloudRunnerDebugEnv(): string | boolean {
return CloudRunnerOptions.getInput(`cloudRunnerDebugEnv`) || false;
static get skipCache(): boolean {
return CloudRunnerOptions.getInput(`skipCache`) === `true`;
}
static get watchCloudRunnerToEnd(): boolean {
if (CloudRunnerOptions.asyncCloudRunner) {
return false;
}
return CloudRunnerOptions.getInput(`watchToEnd`) === 'true' || true;
public static get asyncCloudRunner(): boolean {
return CloudRunnerOptions.getInput('asyncCloudRunner') === 'true';
}
static get asyncCloudRunner(): boolean {
return (CloudRunnerOptions.getInput('asyncCloudRunner') || `false`) === `true` || false;
}
public static get useSharedLargePackages(): boolean {
return (CloudRunnerOptions.getInput(`useSharedLargePackages`) || 'false') === 'true';
public static get useLargePackages(): boolean {
return CloudRunnerOptions.getInput(`useLargePackages`) === `true`;
}
public static get useSharedBuilder(): boolean {
return (CloudRunnerOptions.getInput(`useSharedBuilder`) || 'true') === 'true';
return CloudRunnerOptions.getInput(`useSharedBuilder`) === `true`;
}
public static get useLz4Compression(): boolean {
return (CloudRunnerOptions.getInput(`useLz4Compression`) || 'false') === 'true';
public static get useCompressionStrategy(): boolean {
return CloudRunnerOptions.getInput(`useCompressionStrategy`) === `true`;
}
public static get useCleanupCron(): boolean {
@ -269,24 +267,16 @@ class CloudRunnerOptions {
// Retained Workspace
// ### ### ###
public static get retainWorkspaces(): boolean {
return CloudRunnerOptions.getInput(`retainWorkspaces`) === 'true' || false;
}
static get maxRetainedWorkspaces(): number {
return Number(CloudRunnerOptions.getInput(`maxRetainedWorkspaces`)) || 3;
public static get maxRetainedWorkspaces(): string {
return CloudRunnerOptions.getInput(`maxRetainedWorkspaces`) || `0`;
}
// ### ### ###
// Garbage Collection
// ### ### ###
static get constantGarbageCollection(): boolean {
return CloudRunnerOptions.getInput(`constantGarbageCollection`) === 'true' || true;
}
static get garbageCollectionMaxAge(): number {
return Number(CloudRunnerOptions.getInput(`garbageCollectionMaxAge`)) || 24;
static get garbageMaxAge(): number {
return Number(CloudRunnerOptions.getInput(`garbageMaxAge`)) || 24;
}
}

14
src/model/cloud-runner/services/cloud-runner-query-override.ts → src/model/cloud-runner/options/cloud-runner-query-override.ts
View File

@ -1,6 +1,6 @@
import Input from '../../input';
import { GenericInputReader } from '../../input-readers/generic-input-reader';
import CloudRunnerOptions from '../cloud-runner-options';
import CloudRunnerOptions from './cloud-runner-options';
const formatFunction = (value: string, arguments_: any[]) => {
for (const element of arguments_) {
@ -31,11 +31,11 @@ class CloudRunnerQueryOverride {
}
private static shouldUseOverride(query: string) {
if (CloudRunnerOptions.readInputOverrideCommand() !== '') {
if (CloudRunnerOptions.readInputFromOverrideList() !== '') {
if (CloudRunnerOptions.inputPullCommand !== '') {
if (CloudRunnerOptions.pullInputList.length > 0) {
const doesInclude =
CloudRunnerOptions.readInputFromOverrideList().split(',').includes(query) ||
CloudRunnerOptions.readInputFromOverrideList().split(',').includes(Input.ToEnvVarFormat(query));
CloudRunnerOptions.pullInputList.includes(query) ||
CloudRunnerOptions.pullInputList.includes(Input.ToEnvVarFormat(query));
return doesInclude ? true : false;
} else {
@ -50,12 +50,12 @@ class CloudRunnerQueryOverride {
}
return await GenericInputReader.Run(
formatFunction(CloudRunnerOptions.readInputOverrideCommand(), [{ key: 0, value: query }]),
formatFunction(CloudRunnerOptions.inputPullCommand, [{ key: 0, value: query }]),
);
}
public static async PopulateQueryOverrideInput() {
const queries = CloudRunnerOptions.readInputFromOverrideList().split(',');
const queries = CloudRunnerOptions.pullInputList;
CloudRunnerQueryOverride.queryOverrides = {};
for (const element of queries) {
if (CloudRunnerQueryOverride.shouldUseOverride(element)) {

0
src/model/cloud-runner/services/cloud-runner-secret.ts → src/model/cloud-runner/options/cloud-runner-secret.ts
View File

0
src/model/cloud-runner/cloud-runner-statics.ts → src/model/cloud-runner/options/cloud-runner-statics.ts
View File

6
src/model/cloud-runner/cloud-runner-step-state.ts → src/model/cloud-runner/options/cloud-runner-step-parameters.ts
View File

@ -1,7 +1,7 @@
import CloudRunnerEnvironmentVariable from './services/cloud-runner-environment-variable';
import CloudRunnerSecret from './services/cloud-runner-secret';
import CloudRunnerEnvironmentVariable from './cloud-runner-environment-variable';
import CloudRunnerSecret from './cloud-runner-secret';
export class CloudRunnerStepState {
export class CloudRunnerStepParameters {
public image: string;
public environment: CloudRunnerEnvironmentVariable[];
public secrets: CloudRunnerSecret[];

2
src/model/cloud-runner/providers/aws/aws-base-stack.ts
View File

@ -1,4 +1,4 @@
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import * as core from '@actions/core';
import * as SDK from 'aws-sdk';
import { BaseStackFormation } from './cloud-formations/base-stack-formation';

2
src/model/cloud-runner/providers/aws/aws-error.ts
View File

@ -1,4 +1,4 @@
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import * as SDK from 'aws-sdk';
import * as core from '@actions/core';
import CloudRunner from '../../cloud-runner';

25
src/model/cloud-runner/providers/aws/aws-job-stack.ts
View File

@ -1,12 +1,12 @@
import * as SDK from 'aws-sdk';
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
import CloudRunnerSecret from '../../services/cloud-runner-secret';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import { AWSCloudFormationTemplates } from './aws-cloud-formation-templates';
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { AWSError } from './aws-error';
import CloudRunner from '../../cloud-runner';
import { CleanupCronFormation } from './cloud-formations/cleanup-cron-formation';
import CloudRunnerOptions from '../../cloud-runner-options';
import CloudRunnerOptions from '../../options/cloud-runner-options';
import { TaskDefinitionFormation } from './cloud-formations/task-definition-formation';
export class AWSJobStack {
@ -27,21 +27,19 @@ export class AWSJobStack {
): Promise<CloudRunnerAWSTaskDef> {
const taskDefStackName = `${this.baseStackName}-${buildGuid}`;
let taskDefCloudFormation = AWSCloudFormationTemplates.readTaskCloudFormationTemplate();
const cpu = CloudRunner.buildParameters.cloudRunnerCpu || '1024';
const memory = CloudRunner.buildParameters.cloudRunnerMemory || '3072';
taskDefCloudFormation = taskDefCloudFormation.replace(
`ContainerCpu:
Default: 1024`,
`ContainerCpu:
Default: ${Number.parseInt(cpu)}`,
Default: ${Number.parseInt(CloudRunner.buildParameters.containerCpu)}`,
);
taskDefCloudFormation = taskDefCloudFormation.replace(
`ContainerMemory:
Default: 2048`,
`ContainerMemory:
Default: ${Number.parseInt(memory)}`,
Default: ${Number.parseInt(CloudRunner.buildParameters.containerMemory)}`,
);
if (CloudRunnerOptions.watchCloudRunnerToEnd) {
if (!CloudRunnerOptions.asyncCloudRunner) {
taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
taskDefCloudFormation,
'# template resources logstream',
@ -116,7 +114,7 @@ export class AWSJobStack {
...secretsMappedToCloudFormationParameters,
];
CloudRunnerLogger.log(
`Starting AWS job with memory: ${CloudRunner.buildParameters.cloudRunnerMemory} cpu: ${CloudRunner.buildParameters.cloudRunnerCpu}`,
`Starting AWS job with memory: ${CloudRunner.buildParameters.containerMemory} cpu: ${CloudRunner.buildParameters.containerCpu}`,
);
let previousStackExists = true;
while (previousStackExists) {
@ -140,11 +138,16 @@ export class AWSJobStack {
Capabilities: ['CAPABILITY_IAM'],
Parameters: parameters,
};
try {
CloudRunnerLogger.log(`Creating job aws formation ${taskDefStackName}`);
await CF.createStack(createStackInput).promise();
await CF.waitFor('stackCreateComplete', { StackName: taskDefStackName }).promise();
const describeStack = await CF.describeStacks({ StackName: taskDefStackName }).promise();
for (const parameter of parameters) {
if (!describeStack.Stacks?.[0].Parameters?.some((x) => x.ParameterKey === parameter.ParameterKey)) {
throw new Error(`Parameter ${parameter.ParameterKey} not found in stack`);
}
}
} catch (error) {
await AWSError.handleStackCreationFailure(error, CF, taskDefStackName);
throw error;
@ -180,7 +183,7 @@ export class AWSJobStack {
if (CloudRunnerOptions.useCleanupCron) {
try {
CloudRunnerLogger.log(`Creating job cleanup formation`);
CF.createStack(createCleanupStackInput).promise();
await CF.createStack(createCleanupStackInput).promise();
// await CF.waitFor('stackCreateComplete', { StackName: createCleanupStackInput.StackName }).promise();
} catch (error) {

66
src/model/cloud-runner/providers/aws/aws-task-runner.ts
View File

@ -1,14 +1,14 @@
import * as AWS from 'aws-sdk';
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import * as core from '@actions/core';
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
import * as zlib from 'node:zlib';
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { Input } from '../../..';
import CloudRunner from '../../cloud-runner';
import { CloudRunnerCustomHooks } from '../../services/cloud-runner-custom-hooks';
import { FollowLogStreamService } from '../../services/follow-log-stream-service';
import CloudRunnerOptions from '../../cloud-runner-options';
import { CommandHookService } from '../../services/hooks/command-hook-service';
import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';
import CloudRunnerOptions from '../../options/cloud-runner-options';
import GitHub from '../../../github';
class AWSTaskRunner {
@ -32,7 +32,7 @@ class AWSTaskRunner {
const streamName =
taskDef.taskDefResources?.find((x) => x.LogicalResourceId === 'KinesisStream')?.PhysicalResourceId || '';
const task = await AWSTaskRunner.ECS.runTask({
const runParameters = {
cluster,
taskDefinition,
platformVersion: '1.4.0',
@ -41,7 +41,7 @@ class AWSTaskRunner {
{
name: taskDef.taskDefStackName,
environment,
command: ['-c', CloudRunnerCustomHooks.ApplyHooksToCommands(commands, CloudRunner.buildParameters)],
command: ['-c', CommandHookService.ApplyHooksToCommands(commands, CloudRunner.buildParameters)],
},
],
},
@ -53,16 +53,23 @@ class AWSTaskRunner {
securityGroups: [ContainerSecurityGroup],
},
},
}).promise();
};
if (JSON.stringify(runParameters.overrides.containerOverrides).length > 8192) {
CloudRunnerLogger.log(JSON.stringify(runParameters.overrides.containerOverrides, undefined, 4));
throw new Error(`Container Overrides length must be at most 8192`);
}
const task = await AWSTaskRunner.ECS.runTask(runParameters).promise();
const taskArn = task.tasks?.[0].taskArn || '';
CloudRunnerLogger.log('Cloud runner job is starting');
await AWSTaskRunner.waitUntilTaskRunning(taskArn, cluster);
CloudRunnerLogger.log(
`Cloud runner job status is running ${(await AWSTaskRunner.describeTasks(cluster, taskArn))?.lastStatus} Watch:${
CloudRunnerOptions.watchCloudRunnerToEnd
} Async:${CloudRunnerOptions.asyncCloudRunner}`,
`Cloud runner job status is running ${(await AWSTaskRunner.describeTasks(cluster, taskArn))?.lastStatus} Async:${
CloudRunnerOptions.asyncCloudRunner
}`,
);
if (!CloudRunnerOptions.watchCloudRunnerToEnd) {
if (CloudRunnerOptions.asyncCloudRunner) {
const shouldCleanup: boolean = false;
const output: string = '';
CloudRunnerLogger.log(`Watch Cloud Runner To End: false`);
@ -72,26 +79,31 @@ class AWSTaskRunner {
CloudRunnerLogger.log(`Streaming...`);
const { output, shouldCleanup } = await this.streamLogsUntilTaskStops(cluster, taskArn, streamName);
await new Promise((resolve) => resolve(5000));
const taskData = await AWSTaskRunner.describeTasks(cluster, taskArn);
const containerState = taskData.containers?.[0];
const exitCode = containerState?.exitCode || undefined;
let exitCode;
let containerState;
let taskData;
while (exitCode === undefined) {
await new Promise((resolve) => resolve(10000));
taskData = await AWSTaskRunner.describeTasks(cluster, taskArn);
containerState = taskData.containers?.[0];
exitCode = containerState?.exitCode;
}
CloudRunnerLogger.log(`Container State: ${JSON.stringify(containerState, undefined, 4)}`);
const wasSuccessful = exitCode === 0 || (exitCode === undefined && taskData.lastStatus === 'RUNNING');
if (exitCode === undefined) {
CloudRunnerLogger.logWarning(`Undefined exitcode for container`);
}
const wasSuccessful = exitCode === 0;
if (wasSuccessful) {
CloudRunnerLogger.log(`Cloud runner job has finished successfully`);
return { output, shouldCleanup };
} else {
if (taskData.stoppedReason === 'Essential container in task exited' && exitCode === 1) {
throw new Error('Container exited with code 1');
}
const message = `Cloud runner job exit code ${exitCode}`;
taskData.overrides = undefined;
taskData.attachments = undefined;
CloudRunnerLogger.log(`${message} ${JSON.stringify(taskData, undefined, 4)}`);
throw new Error(message);
}
if (taskData?.stoppedReason === 'Essential container in task exited' && exitCode === 1) {
throw new Error('Container exited with code 1');
}
throw new Error(`Task failed`);
}
private static async waitUntilTaskRunning(taskArn: string, cluster: string) {
@ -129,7 +141,7 @@ class AWSTaskRunner {
const stream = await AWSTaskRunner.getLogStream(kinesisStreamName);
let iterator = await AWSTaskRunner.getLogIterator(stream);
const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${Input.region}#logsV2:log-groups/log-group/${CloudRunner.buildParameters.awsBaseStackName}${AWSTaskRunner.encodedUnderscore}${CloudRunner.buildParameters.awsBaseStackName}-${CloudRunner.buildParameters.buildGuid}`;
const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${Input.region}#logsV2:log-groups/log-group/${CloudRunner.buildParameters.awsStackName}${AWSTaskRunner.encodedUnderscore}${CloudRunner.buildParameters.awsStackName}-${CloudRunner.buildParameters.buildGuid}`;
CloudRunnerLogger.log(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`);
await GitHub.updateGitHubCheck(`You view the log stream on AWS Cloud Watch: ${logBaseUrl}`, ``);
let shouldReadLogs = true;

2
src/model/cloud-runner/providers/aws/cloud-formations/task-definition-formation.ts
View File

@ -30,7 +30,7 @@ Parameters:
Type: Number
Description: How much CPU to give the container. 1024 is 1 CPU
ContainerMemory:
Default: 2048
Default: 4096
Type: Number
Description: How much memory in megabytes to give the container
BUILDGUID:

21
src/model/cloud-runner/providers/aws/index.ts
View File

@ -1,11 +1,11 @@
import * as SDK from 'aws-sdk';
import CloudRunnerSecret from '../../services/cloud-runner-secret';
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
import AwsTaskRunner from './aws-task-runner';
import { ProviderInterface } from '../provider-interface';
import BuildParameters from '../../../build-parameters';
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { AWSJobStack as AwsJobStack } from './aws-job-stack';
import { AWSBaseStack as AwsBaseStack } from './aws-base-stack';
import { Input } from '../../..';
@ -13,13 +13,13 @@ import { GarbageCollectionService } from './services/garbage-collection-service'
import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow';
import { TaskService } from './services/task-service';
import CloudRunnerOptions from '../../cloud-runner-options';
import CloudRunnerOptions from '../../options/cloud-runner-options';
class AWSBuildEnvironment implements ProviderInterface {
private baseStackName: string;
constructor(buildParameters: BuildParameters) {
this.baseStackName = buildParameters.awsBaseStackName;
this.baseStackName = buildParameters.awsStackName;
}
async listResources(): Promise<ProviderResource[]> {
await TaskService.getCloudFormationJobStacks();
@ -75,7 +75,11 @@ class AWSBuildEnvironment implements ProviderInterface {
branchName: string,
// eslint-disable-next-line no-unused-vars
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
) {}
) {
process.env.AWS_REGION = Input.region;
const CF = new SDK.CloudFormation();
await new AwsBaseStack(this.baseStackName).setupBaseStack(CF);
}
async runTaskInWorkflow(
buildGuid: string,
@ -94,8 +98,6 @@ class AWSBuildEnvironment implements ProviderInterface {
CloudRunnerLogger.log(`AWS Region: ${CF.config.region}`);
const entrypoint = ['/bin/sh'];
const startTimeMs = Date.now();
await new AwsBaseStack(this.baseStackName).setupBaseStack(CF);
const taskDef = await new AwsJobStack(this.baseStackName).setupCloudFormations(
CF,
buildGuid,
@ -143,6 +145,9 @@ class AWSBuildEnvironment implements ProviderInterface {
await CF.waitFor('stackDeleteComplete', {
StackName: taskDef.taskDefStackName,
}).promise();
await CF.waitFor('stackDeleteComplete', {
StackName: `${taskDef.taskDefStackName}-cleanup`,
}).promise();
CloudRunnerLogger.log(`Deleted Stack: ${taskDef.taskDefStackName}`);
CloudRunnerLogger.log('Cleanup complete');
}

2
src/model/cloud-runner/providers/aws/services/garbage-collection-service.ts
View File

@ -1,6 +1,6 @@
import AWS from 'aws-sdk';
import Input from '../../../../input';
import CloudRunnerLogger from '../../../services/cloud-runner-logger';
import CloudRunnerLogger from '../../../services/core/cloud-runner-logger';
import { TaskService } from './task-service';
export class GarbageCollectionService {

4
src/model/cloud-runner/providers/aws/services/task-service.ts
View File

@ -1,6 +1,6 @@
import AWS from 'aws-sdk';
import Input from '../../../../input';
import CloudRunnerLogger from '../../../services/cloud-runner-logger';
import CloudRunnerLogger from '../../../services/core/cloud-runner-logger';
import { BaseStackFormation } from '../cloud-formations/base-stack-formation';
import AwsTaskRunner from '../aws-task-runner';
import { ListObjectsRequest } from 'aws-sdk/clients/s3';
@ -161,7 +161,7 @@ export class TaskService {
process.env.AWS_REGION = Input.region;
const s3 = new AWS.S3();
const listRequest: ListObjectsRequest = {
Bucket: CloudRunner.buildParameters.awsBaseStackName,
Bucket: CloudRunner.buildParameters.awsStackName,
};
const results = await s3.listObjects(listRequest).promise();

22
src/model/cloud-runner/providers/docker/index.ts
View File

@ -1,20 +1,21 @@
import BuildParameters from '../../../build-parameters';
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { ProviderInterface } from '../provider-interface';
import CloudRunnerSecret from '../../services/cloud-runner-secret';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import Docker from '../../../docker';
import { Action } from '../../..';
import { writeFileSync } from 'fs';
import { writeFileSync } from 'node:fs';
import CloudRunner from '../../cloud-runner';
import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow';
import { CloudRunnerSystem } from '../../services/cloud-runner-system';
import fs from 'node:fs';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
import * as fs from 'node:fs';
import { CommandHookService } from '../../services/hooks/command-hook-service';
import { StringKeyValuePair } from '../../../shared-types';
class LocalDockerCloudRunner implements ProviderInterface {
public buildParameters: BuildParameters | undefined;
public buildParameters!: BuildParameters;
listResources(): Promise<ProviderResource[]> {
return new Promise((resolve) => resolve([]));
@ -51,14 +52,14 @@ class LocalDockerCloudRunner implements ProviderInterface {
if (
fs.existsSync(
`${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
}`,
)
) {
await CloudRunnerSystem.Run(`ls ${workspace}/cloud-runner-cache/cache/build/`);
await CloudRunnerSystem.Run(
`rm -r ${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
}`,
);
}
@ -118,7 +119,7 @@ set -e
mkdir -p /github/workspace/cloud-runner-cache
mkdir -p /data/cache
cp -a /github/workspace/cloud-runner-cache/. ${sharedFolder}
${commands}
${CommandHookService.ApplyHooksToCommands(commands, this.buildParameters)}
cp -a ${sharedFolder}. /github/workspace/cloud-runner-cache/
`;
writeFileSync(`${workspace}/${entrypointFilePath}`, fileContents, {
@ -149,6 +150,7 @@ cp -a ${sharedFolder}. /github/workspace/cloud-runner-cache/
},
},
true,
false,
);
return myOutput;

153
src/model/cloud-runner/providers/k8s/index.ts
View File

@ -2,19 +2,18 @@ import * as k8s from '@kubernetes/client-node';
import { BuildParameters } from '../../..';
import * as core from '@actions/core';
import { ProviderInterface } from '../provider-interface';
import CloudRunnerSecret from '../../services/cloud-runner-secret';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import KubernetesStorage from './kubernetes-storage';
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import KubernetesTaskRunner from './kubernetes-task-runner';
import KubernetesSecret from './kubernetes-secret';
import KubernetesJobSpecFactory from './kubernetes-job-spec-factory';
import KubernetesServiceAccount from './kubernetes-service-account';
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { CoreV1Api } from '@kubernetes/client-node';
import CloudRunner from '../../cloud-runner';
import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow';
import KubernetesPods from './kubernetes-pods';
class Kubernetes implements ProviderInterface {
public static Instance: Kubernetes;
@ -94,16 +93,8 @@ class Kubernetes implements ProviderInterface {
) {
try {
this.buildParameters = buildParameters;
const id = buildParameters.retainWorkspace ? CloudRunner.lockedWorkspace : buildParameters.buildGuid;
this.pvcName = `unity-builder-pvc-${id}`;
this.cleanupCronJobName = `unity-builder-cronjob-${id}`;
this.cleanupCronJobName = `unity-builder-cronjob-${buildParameters.buildGuid}`;
this.serviceAccountName = `service-account-${buildParameters.buildGuid}`;
await KubernetesStorage.createPersistentVolumeClaim(
buildParameters,
this.pvcName,
this.kubeClient,
this.namespace,
);
await KubernetesServiceAccount.createServiceAccount(this.serviceAccountName, this.namespace, this.kubeClient);
} catch (error) {
@ -124,74 +115,99 @@ class Kubernetes implements ProviderInterface {
CloudRunnerLogger.log('Cloud Runner K8s workflow!');
// Setup
const id = BuildParameters.shouldUseRetainedWorkspaceMode(this.buildParameters)
? CloudRunner.lockedWorkspace
: this.buildParameters.buildGuid;
this.pvcName = `unity-builder-pvc-${id}`;
await KubernetesStorage.createPersistentVolumeClaim(
this.buildParameters,
this.pvcName,
this.kubeClient,
this.namespace,
);
this.buildGuid = buildGuid;
this.secretName = `build-credentials-${this.buildGuid}`;
this.jobName = `unity-builder-job-${this.buildGuid}`;
this.containerName = `main`;
await KubernetesSecret.createSecret(secrets, this.secretName, this.namespace, this.kubeClient);
await this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets);
this.setPodNameAndContainerName(await Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace));
CloudRunnerLogger.log('Watching pod until running');
await KubernetesTaskRunner.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
let output = '';
// eslint-disable-next-line no-constant-condition
while (true) {
try {
CloudRunnerLogger.log('Pod running, streaming logs');
output = await KubernetesTaskRunner.runTask(
this.kubeConfig,
this.kubeClient,
this.jobName,
this.podName,
'main',
this.namespace,
);
const running = await KubernetesPods.IsPodRunning(this.podName, this.namespace, this.kubeClient);
try {
CloudRunnerLogger.log('Job does not exist');
await this.createJob(commands, image, mountdir, workingdir, environment, secrets);
CloudRunnerLogger.log('Watching pod until running');
await KubernetesTaskRunner.watchUntilPodRunning(this.kubeClient, this.podName, this.namespace);
if (!running) {
CloudRunnerLogger.log(`Pod not found, assumed ended!`);
break;
} else {
CloudRunnerLogger.log('Pod still running, recovering stream...');
}
await this.cleanupTaskResources();
} catch (error: any) {
let errorParsed;
try {
errorParsed = JSON.parse(error);
} catch {
errorParsed = error;
}
const reason = errorParsed.reason || errorParsed.response?.body?.reason || ``;
const errorMessage = errorParsed.message || reason;
const continueStreaming =
errorMessage.includes(`dial timeout, backstop`) ||
errorMessage.includes(`HttpError: HTTP request failed`) ||
errorMessage.includes(`an error occurred when try to find container`) ||
errorMessage.includes(`not found`) ||
errorMessage.includes(`Not Found`);
if (continueStreaming) {
CloudRunnerLogger.log('Log Stream Container Not Found');
await new Promise((resolve) => resolve(5000));
continue;
} else {
CloudRunnerLogger.log(`error running k8s workflow ${error}`);
throw error;
}
}
CloudRunnerLogger.log('Pod running, streaming logs');
CloudRunnerLogger.log(
`Starting logs follow for pod: ${this.podName} container: ${this.containerName} namespace: ${this.namespace} pvc: ${this.pvcName} ${CloudRunner.buildParameters.kubeVolumeSize}/${CloudRunner.buildParameters.containerCpu}/${CloudRunner.buildParameters.containerMemory}`,
);
output += await KubernetesTaskRunner.runTask(
this.kubeConfig,
this.kubeClient,
this.jobName,
this.podName,
this.containerName,
this.namespace,
);
} catch (error: any) {
CloudRunnerLogger.log(`error running k8s workflow ${error}`);
await new Promise((resolve) => setTimeout(resolve, 3000));
CloudRunnerLogger.log(
JSON.stringify(
(await this.kubeClient.listNamespacedEvent(this.namespace)).body.items
.map((x) => {
return {
message: x.message || ``,
name: x.metadata.name || ``,
reason: x.reason || ``,
};
})
.filter((x) => x.name.includes(this.podName)),
undefined,
4,
),
);
await this.cleanupTaskResources();
throw error;
}
await this.cleanupTaskResources();
return output;
} catch (error) {
CloudRunnerLogger.log('Running job failed');
core.error(JSON.stringify(error, undefined, 4));
await this.cleanupTaskResources();
// await this.cleanupTaskResources();
throw error;
}
}
private async createJob(
commands: string,
image: string,
mountdir: string,
workingdir: string,
environment: CloudRunnerEnvironmentVariable[],
secrets: CloudRunnerSecret[],
) {
await this.createNamespacedJob(commands, image, mountdir, workingdir, environment, secrets);
const find = await Kubernetes.findPodFromJob(this.kubeClient, this.jobName, this.namespace);
this.setPodNameAndContainerName(find);
}
private async doesJobExist(name: string) {
const jobs = await this.kubeClientBatch.listNamespacedJob(this.namespace);
return jobs.body.items.some((x) => x.metadata?.name === name);
}
private async doesFailedJobExist() {
const podStatus = await this.kubeClient.readNamespacedPodStatus(this.podName, this.namespace);
return podStatus.body.status?.phase === `Failed`;
}
private async createNamespacedJob(
commands: string,
image: string,
@ -215,14 +231,15 @@ class Kubernetes implements ProviderInterface {
this.pvcName,
this.jobName,
k8s,
this.containerName,
);
await new Promise((promise) => setTimeout(promise, 15000));
await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec);
const result = await this.kubeClientBatch.createNamespacedJob(this.namespace, jobSpec);
CloudRunnerLogger.log(`Build job created`);
await new Promise((promise) => setTimeout(promise, 5000));
CloudRunnerLogger.log('Job created');
return;
return result.body.metadata?.name;
} catch (error) {
CloudRunnerLogger.log(`Error occured creating job: ${error}`);
throw error;
@ -232,7 +249,7 @@ class Kubernetes implements ProviderInterface {
setPodNameAndContainerName(pod: k8s.V1Pod) {
this.podName = pod.metadata?.name || '';
this.containerName = pod.status?.containerStatuses?.[0].name || '';
this.containerName = pod.status?.containerStatuses?.[0].name || this.containerName;
}
async cleanupTaskResources() {
@ -265,7 +282,7 @@ class Kubernetes implements ProviderInterface {
// eslint-disable-next-line no-unused-vars
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
) {
if (buildParameters.retainWorkspace) {
if (BuildParameters.shouldUseRetainedWorkspaceMode(buildParameters)) {
return;
}
CloudRunnerLogger.log(`deleting PVC`);

80
src/model/cloud-runner/providers/k8s/kubernetes-job-spec-factory.ts
View File

@ -1,8 +1,8 @@
import { V1EnvVar, V1EnvVarSource, V1SecretKeySelector } from '@kubernetes/client-node';
import BuildParameters from '../../../build-parameters';
import { CloudRunnerCustomHooks } from '../../services/cloud-runner-custom-hooks';
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
import CloudRunnerSecret from '../../services/cloud-runner-secret';
import { CommandHookService } from '../../services/hooks/command-hook-service';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import CloudRunner from '../../cloud-runner';
class KubernetesJobSpecFactory {
@ -19,63 +19,8 @@ class KubernetesJobSpecFactory {
pvcName: string,
jobName: string,
k8s: any,
containerName: string,
) {
environment.push(
...[
{
name: 'GITHUB_SHA',
value: buildGuid,
},
{
name: 'GITHUB_WORKSPACE',
value: '/data/repo',
},
{
name: 'PROJECT_PATH',
value: buildParameters.projectPath,
},
{
name: 'BUILD_PATH',
value: buildParameters.buildPath,
},
{
name: 'BUILD_FILE',
value: buildParameters.buildFile,
},
{
name: 'BUILD_NAME',
value: buildParameters.buildName,
},
{
name: 'BUILD_METHOD',
value: buildParameters.buildMethod,
},
{
name: 'CUSTOM_PARAMETERS',
value: buildParameters.customParameters,
},
{
name: 'CHOWN_FILES_TO',
value: buildParameters.chownFilesTo,
},
{
name: 'BUILD_TARGET',
value: buildParameters.targetPlatform,
},
{
name: 'ANDROID_VERSION_CODE',
value: buildParameters.androidVersionCode.toString(),
},
{
name: 'ANDROID_KEYSTORE_NAME',
value: buildParameters.androidKeystoreName,
},
{
name: 'ANDROID_KEYALIAS_NAME',
value: buildParameters.androidKeyaliasName,
},
],
);
const job = new k8s.V1Job();
job.apiVersion = 'batch/v1';
job.kind = 'Job';
@ -87,6 +32,7 @@ class KubernetesJobSpecFactory {
},
};
job.spec = {
ttlSecondsAfterFinished: 9999,
backoffLimit: 0,
template: {
spec: {
@ -100,16 +46,20 @@ class KubernetesJobSpecFactory {
],
containers: [
{
name: 'main',
ttlSecondsAfterFinished: 9999,
name: containerName,
image,
command: ['/bin/sh'],
args: ['-c', CloudRunnerCustomHooks.ApplyHooksToCommands(command, CloudRunner.buildParameters)],
args: [
'-c',
`${CommandHookService.ApplyHooksToCommands(`${command}\nsleep 2m`, CloudRunner.buildParameters)}`,
],
workingDir: `${workingDirectory}`,
resources: {
requests: {
memory: buildParameters.cloudRunnerMemory || '750M',
cpu: buildParameters.cloudRunnerCpu || '1',
memory: `${Number.parseInt(buildParameters.containerMemory) / 1024}G` || '750M',
cpu: Number.parseInt(buildParameters.containerCpu) / 1024 || '1',
},
},
env: [
@ -135,7 +85,7 @@ class KubernetesJobSpecFactory {
volumeMounts: [
{
name: 'build-mount',
mountPath: `/${mountdir}`,
mountPath: `${mountdir}`,
},
],
lifecycle: {
@ -158,7 +108,7 @@ class KubernetesJobSpecFactory {
},
};
job.spec.template.spec.containers[0].resources.requests[`ephemeral-storage`] = '5Gi';
job.spec.template.spec.containers[0].resources.requests[`ephemeral-storage`] = '10Gi';
return job;
}

8
src/model/cloud-runner/providers/k8s/kubernetes-pods.ts
View File

@ -1,4 +1,4 @@
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { CoreV1Api } from '@kubernetes/client-node';
class KubernetesPods {
public static async IsPodRunning(podName: string, namespace: string, kubeClient: CoreV1Api) {
@ -12,6 +12,12 @@ class KubernetesPods {
return running;
}
public static async GetPodStatus(podName: string, namespace: string, kubeClient: CoreV1Api) {
const pods = (await kubeClient.listNamespacedPod(namespace)).body.items.find((x) => podName === x.metadata?.name);
const phase = pods?.status?.phase || 'undefined status';
return phase;
}
}
export default KubernetesPods;

4
src/model/cloud-runner/providers/k8s/kubernetes-secret.ts
View File

@ -1,7 +1,7 @@
import { CoreV1Api } from '@kubernetes/client-node';
import CloudRunnerSecret from '../../services/cloud-runner-secret';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import * as k8s from '@kubernetes/client-node';
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import * as base64 from 'base-64';
class KubernetesSecret {

2
src/model/cloud-runner/providers/k8s/kubernetes-storage.ts
View File

@ -2,7 +2,7 @@ import waitUntil from 'async-wait-until';
import * as core from '@actions/core';
import * as k8s from '@kubernetes/client-node';
import BuildParameters from '../../../build-parameters';
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { IncomingMessage } from 'node:http';
import GitHub from '../../../github';

168
src/model/cloud-runner/providers/k8s/kubernetes-task-runner.ts
View File

@ -1,12 +1,15 @@
import { CoreV1Api, KubeConfig, Log } from '@kubernetes/client-node';
import { Writable } from 'stream';
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import * as core from '@actions/core';
import { CloudRunnerStatics } from '../../cloud-runner-statics';
import { CoreV1Api, KubeConfig } from '@kubernetes/client-node';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import waitUntil from 'async-wait-until';
import { FollowLogStreamService } from '../../services/follow-log-stream-service';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
import CloudRunner from '../../cloud-runner';
import KubernetesPods from './kubernetes-pods';
import { FollowLogStreamService } from '../../services/core/follow-log-stream-service';
class KubernetesTaskRunner {
static lastReceivedTimestamp: number = 0;
static readonly maxRetry: number = 3;
static lastReceivedMessage: string = ``;
static async runTask(
kubeConfig: KubeConfig,
kubeClient: CoreV1Api,
@ -15,84 +18,120 @@ class KubernetesTaskRunner {
containerName: string,
namespace: string,
) {
CloudRunnerLogger.log(`Streaming logs from pod: ${podName} container: ${containerName} namespace: ${namespace}`);
const stream = new Writable();
let output = '';
let didStreamAnyLogs: boolean = false;
let shouldReadLogs = true;
let shouldCleanup = true;
stream._write = (chunk, encoding, next) => {
didStreamAnyLogs = true;
let message = chunk.toString().trimRight(`\n`);
message = `[${CloudRunnerStatics.logPrefix}] ${message}`;
({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
message,
shouldReadLogs,
shouldCleanup,
output,
));
next();
};
const logOptions = {
follow: true,
pretty: false,
previous: false,
};
try {
const resultError = await new Promise((resolve) =>
new Log(kubeConfig).log(namespace, podName, containerName, stream, resolve, logOptions),
let sinceTime = ``;
let retriesAfterFinish = 0;
// eslint-disable-next-line no-constant-condition
while (true) {
await new Promise((resolve) => setTimeout(resolve, 3000));
const lastReceivedMessage =
KubernetesTaskRunner.lastReceivedTimestamp > 0
? `\nLast Log Message "${this.lastReceivedMessage}" ${this.lastReceivedTimestamp}`
: ``;
CloudRunnerLogger.log(
`Streaming logs from pod: ${podName} container: ${containerName} namespace: ${namespace} ${CloudRunner.buildParameters.kubeVolumeSize}/${CloudRunner.buildParameters.containerCpu}/${CloudRunner.buildParameters.containerMemory}\n${lastReceivedMessage}`,
);
stream.destroy();
if (resultError) {
throw resultError;
if (KubernetesTaskRunner.lastReceivedTimestamp > 0) {
const currentDate = new Date(KubernetesTaskRunner.lastReceivedTimestamp);
const dateTimeIsoString = currentDate.toISOString();
sinceTime = ` --since-time="${dateTimeIsoString}"`;
}
if (!didStreamAnyLogs) {
core.error('Failed to stream any logs, listing namespace events, check for an error with the container');
core.error(
JSON.stringify(
{
events: (await kubeClient.listNamespacedEvent(namespace)).body.items
.filter((x) => {
return x.involvedObject.name === podName || x.involvedObject.name === jobName;
})
.map((x) => {
return {
type: x.involvedObject.kind,
name: x.involvedObject.name,
message: x.message,
};
}),
},
undefined,
4,
),
let extraFlags = ``;
extraFlags += (await KubernetesPods.IsPodRunning(podName, namespace, kubeClient))
? ` -f -c ${containerName}`
: ` --previous`;
let lastMessageSeenIncludedInChunk = false;
let lastMessageSeen = false;
let logs;
try {
logs = await CloudRunnerSystem.Run(
`kubectl logs ${podName}${extraFlags} --timestamps${sinceTime}`,
false,
true,
);
throw new Error(`No logs streamed from k8s`);
} catch (error: any) {
await new Promise((resolve) => setTimeout(resolve, 3000));
const continueStreaming = await KubernetesPods.IsPodRunning(podName, namespace, kubeClient);
CloudRunnerLogger.log(`K8s logging error ${error} ${continueStreaming}`);
if (continueStreaming) {
continue;
}
if (retriesAfterFinish < KubernetesTaskRunner.maxRetry) {
retriesAfterFinish++;
continue;
}
throw error;
}
} catch (error) {
if (stream) {
stream.destroy();
const splitLogs = logs.split(`\n`);
for (const chunk of splitLogs) {
if (
chunk.replace(/\s/g, ``) === KubernetesTaskRunner.lastReceivedMessage.replace(/\s/g, ``) &&
KubernetesTaskRunner.lastReceivedMessage.replace(/\s/g, ``) !== ``
) {
CloudRunnerLogger.log(`Previous log message found ${chunk}`);
lastMessageSeenIncludedInChunk = true;
}
}
for (const chunk of splitLogs) {
const newDate = Date.parse(`${chunk.toString().split(`Z `)[0]}Z`);
if (chunk.replace(/\s/g, ``) === KubernetesTaskRunner.lastReceivedMessage.replace(/\s/g, ``)) {
lastMessageSeen = true;
}
if (lastMessageSeenIncludedInChunk && !lastMessageSeen) {
continue;
}
const message = CloudRunner.buildParameters.cloudRunnerDebug ? chunk : chunk.split(`Z `)[1];
KubernetesTaskRunner.lastReceivedMessage = chunk;
KubernetesTaskRunner.lastReceivedTimestamp = newDate;
({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
message,
shouldReadLogs,
shouldCleanup,
output,
));
}
if (FollowLogStreamService.DidReceiveEndOfTransmission) {
CloudRunnerLogger.log('end of log stream');
break;
}
throw error;
}
CloudRunnerLogger.log('end of log stream');
return output;
}
static async watchUntilPodRunning(kubeClient: CoreV1Api, podName: string, namespace: string) {
let success: boolean = false;
let message = ``;
CloudRunnerLogger.log(`Watching ${podName} ${namespace}`);
await waitUntil(
async () => {
const status = await kubeClient.readNamespacedPodStatus(podName, namespace);
const phase = status?.body.status?.phase;
success = phase === 'Running';
CloudRunnerLogger.log(
`${status.body.status?.phase} ${status.body.status?.conditions?.[0].reason || ''} ${
status.body.status?.conditions?.[0].message || ''
}`,
);
message = `Phase:${status.body.status?.phase} \n Reason:${
status.body.status?.conditions?.[0].reason || ''
} \n Message:${status.body.status?.conditions?.[0].message || ''}`;
// CloudRunnerLogger.log(
// JSON.stringify(
// (await kubeClient.listNamespacedEvent(namespace)).body.items
// .map((x) => {
// return {
// message: x.message || ``,
// name: x.metadata.name || ``,
// reason: x.reason || ``,
// };
// })
// .filter((x) => x.name.includes(podName)),
// undefined,
// 4,
// ),
// );
if (success || phase !== 'Pending') return true;
return false;
@ -102,6 +141,9 @@ class KubernetesTaskRunner {
intervalBetweenAttempts: 15000,
},
);
if (!success) {
CloudRunnerLogger.log(message);
}
return success;
}

8
src/model/cloud-runner/providers/local/index.ts
View File

@ -1,9 +1,9 @@
import BuildParameters from '../../../build-parameters';
import { CloudRunnerSystem } from '../../services/cloud-runner-system';
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { ProviderInterface } from '../provider-interface';
import CloudRunnerSecret from '../../services/cloud-runner-secret';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow';

4
src/model/cloud-runner/providers/provider-interface.ts
View File

@ -1,6 +1,6 @@
import BuildParameters from '../../build-parameters';
import CloudRunnerEnvironmentVariable from '../services/cloud-runner-environment-variable';
import CloudRunnerSecret from '../services/cloud-runner-secret';
import CloudRunnerEnvironmentVariable from '../options/cloud-runner-environment-variable';
import CloudRunnerSecret from '../options/cloud-runner-secret';
import { ProviderResource } from './provider-resource';
import { ProviderWorkflow } from './provider-workflow';

6
src/model/cloud-runner/providers/test/index.ts
View File

@ -1,8 +1,8 @@
import BuildParameters from '../../../build-parameters';
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
import CloudRunnerLogger from '../../services/cloud-runner-logger';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { ProviderInterface } from '../provider-interface';
import CloudRunnerSecret from '../../services/cloud-runner-secret';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow';

29
src/model/cloud-runner/remote-client/caching.ts
View File

@ -2,10 +2,10 @@ import { assert } from 'node:console';
import fs from 'node:fs';
import path from 'node:path';
import CloudRunner from '../cloud-runner';
import CloudRunnerLogger from '../services/cloud-runner-logger';
import { CloudRunnerFolders } from '../services/cloud-runner-folders';
import { CloudRunnerSystem } from '../services/cloud-runner-system';
import { LfsHashing } from '../services/lfs-hashing';
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
import { CloudRunnerFolders } from '../options/cloud-runner-folders';
import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
import { LfsHashing } from '../services/utility/lfs-hashing';
import { RemoteClientLogger } from './remote-client-logger';
import { Cli } from '../../cli/cli';
import { CliFunction } from '../../cli/cli-functions-repository';
@ -44,20 +44,21 @@ export class Caching {
}
public static async PushToCache(cacheFolder: string, sourceFolder: string, cacheArtifactName: string) {
CloudRunnerLogger.log(`Pushing to cache ${sourceFolder}`);
cacheArtifactName = cacheArtifactName.replace(' ', '');
const startPath = process.cwd();
let compressionSuffix = '';
if (CloudRunner.buildParameters.useLz4Compression === true) {
if (CloudRunner.buildParameters.useCompressionStrategy === true) {
compressionSuffix = `.lz4`;
}
CloudRunnerLogger.log(`Compression: ${CloudRunner.buildParameters.useLz4Compression} ${compressionSuffix}`);
CloudRunnerLogger.log(`Compression: ${CloudRunner.buildParameters.useCompressionStrategy} ${compressionSuffix}`);
try {
if (!(await fileExists(cacheFolder))) {
await CloudRunnerSystem.Run(`mkdir -p ${cacheFolder}`);
}
process.chdir(path.resolve(sourceFolder, '..'));
if (CloudRunner.buildParameters.cloudRunnerDebug) {
if (CloudRunner.buildParameters.cloudRunnerDebug === true) {
CloudRunnerLogger.log(
`Hashed cache folder ${await LfsHashing.hashAllFiles(sourceFolder)} ${sourceFolder} ${path.basename(
sourceFolder,
@ -69,11 +70,6 @@ export class Caching {
`There is ${contents.length} files/dir in the source folder ${path.basename(sourceFolder)}`,
);
if (CloudRunner.buildParameters.cloudRunnerDebug) {
// await CloudRunnerSystem.Run(`tree -L 2 ./..`);
// await CloudRunnerSystem.Run(`tree -L 2`);
}
if (contents.length === 0) {
CloudRunnerLogger.log(
`Did not push source folder to cache because it was empty ${path.basename(sourceFolder)}`,
@ -102,9 +98,15 @@ export class Caching {
process.chdir(`${startPath}`);
}
public static async PullFromCache(cacheFolder: string, destinationFolder: string, cacheArtifactName: string = ``) {
CloudRunnerLogger.log(`Pulling from cache ${destinationFolder} ${CloudRunner.buildParameters.skipCache}`);
if (`${CloudRunner.buildParameters.skipCache}` === `true`) {
CloudRunnerLogger.log(`Skipping cache debugSkipCache is true`);
return;
}
cacheArtifactName = cacheArtifactName.replace(' ', '');
let compressionSuffix = '';
if (CloudRunner.buildParameters.useLz4Compression === true) {
if (CloudRunner.buildParameters.useCompressionStrategy === true) {
compressionSuffix = `.lz4`;
}
const startPath = process.cwd();
@ -160,7 +162,6 @@ export class Caching {
RemoteClientLogger.logWarning(
`cache item ${cacheArtifactName}.tar${compressionSuffix} doesn't exist ${destinationFolder}`,
);
await CloudRunnerSystem.Run(`tree ${cacheFolder}`);
throw new Error(`Failed to get cache item, but cache hit was found: ${cacheSelection}`);
}
}

232
src/model/cloud-runner/remote-client/index.ts
View File

@ -1,56 +1,80 @@
import fs from 'node:fs';
import CloudRunner from '../cloud-runner';
import { CloudRunnerFolders } from '../services/cloud-runner-folders';
import { CloudRunnerFolders } from '../options/cloud-runner-folders';
import { Caching } from './caching';
import { LfsHashing } from '../services/lfs-hashing';
import { LfsHashing } from '../services/utility/lfs-hashing';
import { RemoteClientLogger } from './remote-client-logger';
import path from 'node:path';
import { assert } from 'node:console';
import CloudRunnerLogger from '../services/cloud-runner-logger';
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
import { CliFunction } from '../../cli/cli-functions-repository';
import { CloudRunnerSystem } from '../services/cloud-runner-system';
import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
import YAML from 'yaml';
import GitHub from '../../github';
import BuildParameters from '../../build-parameters';
export class RemoteClient {
public static async bootstrapRepository() {
try {
await CloudRunnerSystem.Run(`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute)}`);
await CloudRunnerSystem.Run(
`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderForCacheKeyFull)}`,
);
process.chdir(CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute));
await RemoteClient.cloneRepoWithoutLFSFiles();
RemoteClient.replaceLargePackageReferencesWithSharedReferences();
await RemoteClient.sizeOfFolder(
'repo before lfs cache pull',
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute),
);
const lfsHashes = await LfsHashing.createLFSHashFiles();
if (fs.existsSync(CloudRunnerFolders.libraryFolderAbsolute)) {
RemoteClientLogger.logWarning(`!Warning!: The Unity library was included in the git repository`);
}
await Caching.PullFromCache(
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull),
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute),
`${lfsHashes.lfsGuidSum}`,
);
await RemoteClient.sizeOfFolder('repo after lfs cache pull', CloudRunnerFolders.repoPathAbsolute);
await RemoteClient.pullLatestLFS();
await RemoteClient.sizeOfFolder('repo before lfs git pull', CloudRunnerFolders.repoPathAbsolute);
await Caching.PushToCache(
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull),
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute),
`${lfsHashes.lfsGuidSum}`,
);
await Caching.PullFromCache(
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryCacheFolderFull),
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute),
);
await RemoteClient.sizeOfFolder('repo after library cache pull', CloudRunnerFolders.repoPathAbsolute);
await Caching.handleCachePurging();
} catch (error) {
throw error;
@CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
static async runRemoteClientJob() {
CloudRunnerLogger.log(`bootstrap game ci cloud runner...`);
if (!(await RemoteClient.handleRetainedWorkspace())) {
await RemoteClient.bootstrapRepository();
}
await RemoteClient.runCustomHookFiles(`before-build`);
}
static async runCustomHookFiles(hookLifecycle: string) {
RemoteClientLogger.log(`RunCustomHookFiles: ${hookLifecycle}`);
const gameCiCustomHooksPath = path.join(CloudRunnerFolders.repoPathAbsolute, `game-ci`, `hooks`);
try {
const files = fs.readdirSync(gameCiCustomHooksPath);
for (const file of files) {
const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
const fileContentsObject = YAML.parse(fileContents.toString());
if (fileContentsObject.hook === hookLifecycle) {
RemoteClientLogger.log(`Active Hook File ${file} \n \n file contents: \n ${fileContents}`);
await CloudRunnerSystem.Run(fileContentsObject.commands);
}
}
} catch (error) {
RemoteClientLogger.log(JSON.stringify(error, undefined, 4));
}
}
public static async bootstrapRepository() {
await CloudRunnerSystem.Run(
`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)}`,
);
await CloudRunnerSystem.Run(
`mkdir -p ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.cacheFolderForCacheKeyFull)}`,
);
await RemoteClient.cloneRepoWithoutLFSFiles();
await RemoteClient.replaceLargePackageReferencesWithSharedReferences();
await RemoteClient.sizeOfFolder(
'repo before lfs cache pull',
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute),
);
const lfsHashes = await LfsHashing.createLFSHashFiles();
if (fs.existsSync(CloudRunnerFolders.libraryFolderAbsolute)) {
RemoteClientLogger.logWarning(`!Warning!: The Unity library was included in the git repository`);
}
await Caching.PullFromCache(
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull),
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute),
`${lfsHashes.lfsGuidSum}`,
);
await RemoteClient.sizeOfFolder('repo after lfs cache pull', CloudRunnerFolders.repoPathAbsolute);
await RemoteClient.pullLatestLFS();
await RemoteClient.sizeOfFolder('repo before lfs git pull', CloudRunnerFolders.repoPathAbsolute);
await Caching.PushToCache(
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsCacheFolderFull),
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.lfsFolderAbsolute),
`${lfsHashes.lfsGuidSum}`,
);
await Caching.PullFromCache(
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryCacheFolderFull),
CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.libraryFolderAbsolute),
);
await RemoteClient.sizeOfFolder('repo after library cache pull', CloudRunnerFolders.repoPathAbsolute);
await Caching.handleCachePurging();
}
private static async sizeOfFolder(message: string, folder: string) {
@ -62,58 +86,68 @@ export class RemoteClient {
private static async cloneRepoWithoutLFSFiles() {
process.chdir(`${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
if (
fs.existsSync(CloudRunnerFolders.repoPathAbsolute) &&
!fs.existsSync(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`))
) {
await CloudRunnerSystem.Run(`rm -r ${CloudRunnerFolders.repoPathAbsolute}`);
CloudRunnerLogger.log(`${CloudRunnerFolders.repoPathAbsolute} repo exists, but no git folder, cleaning up`);
}
if (
CloudRunner.buildParameters.retainWorkspace &&
BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters) &&
fs.existsSync(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`))
) {
process.chdir(CloudRunnerFolders.repoPathAbsolute);
RemoteClientLogger.log(
`${CloudRunnerFolders.repoPathAbsolute} repo exists - skipping clone - retained workspace mode ${CloudRunner.buildParameters.retainWorkspace}`,
`${
CloudRunnerFolders.repoPathAbsolute
} repo exists - skipping clone - retained workspace mode ${BuildParameters.shouldUseRetainedWorkspaceMode(
CloudRunner.buildParameters,
)}`,
);
await CloudRunnerSystem.Run(`git fetch && git reset --hard ${CloudRunner.buildParameters.gitSha}`);
return;
}
if (fs.existsSync(CloudRunnerFolders.repoPathAbsolute)) {
RemoteClientLogger.log(`${CloudRunnerFolders.repoPathAbsolute} repo exists cleaning up`);
await CloudRunnerSystem.Run(`rm -r ${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute)}`);
}
RemoteClientLogger.log(`Initializing source repository for cloning with caching of LFS files`);
await CloudRunnerSystem.Run(`git config --global advice.detachedHead false`);
RemoteClientLogger.log(`Cloning the repository being built:`);
await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge --skip -- %f"`);
await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process --skip"`);
try {
RemoteClientLogger.log(`Initializing source repository for cloning with caching of LFS files`);
await CloudRunnerSystem.Run(`git config --global advice.detachedHead false`);
RemoteClientLogger.log(`Cloning the repository being built:`);
await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge --skip -- %f"`);
await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process --skip"`);
await CloudRunnerSystem.Run(
`git clone -q ${CloudRunnerFolders.targetBuildRepoUrl} ${path.basename(CloudRunnerFolders.repoPathAbsolute)}`,
`git clone ${CloudRunnerFolders.targetBuildRepoUrl} ${path.basename(CloudRunnerFolders.repoPathAbsolute)}`,
);
process.chdir(CloudRunnerFolders.repoPathAbsolute);
await CloudRunnerSystem.Run(`git lfs install`);
assert(fs.existsSync(`.git`), 'git folder exists');
RemoteClientLogger.log(`${CloudRunner.buildParameters.branch}`);
await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.branch}`);
await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.gitSha}`);
assert(fs.existsSync(path.join(`.git`, `lfs`)), 'LFS folder should not exist before caching');
RemoteClientLogger.log(`Checked out ${CloudRunner.buildParameters.branch}`);
} catch (error) {
await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
} catch (error: any) {
throw error;
}
process.chdir(CloudRunnerFolders.repoPathAbsolute);
await CloudRunnerSystem.Run(`git lfs install`);
assert(fs.existsSync(`.git`), 'git folder exists');
RemoteClientLogger.log(`${CloudRunner.buildParameters.branch}`);
if (CloudRunner.buildParameters.gitSha !== undefined) {
await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.gitSha}`);
} else {
await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.branch}`);
RemoteClientLogger.log(`buildParameter Git Sha is empty`);
}
assert(fs.existsSync(path.join(`.git`, `lfs`)), 'LFS folder should not exist before caching');
RemoteClientLogger.log(`Checked out ${CloudRunner.buildParameters.branch}`);
}
static replaceLargePackageReferencesWithSharedReferences() {
if (CloudRunner.buildParameters.useSharedLargePackages) {
static async replaceLargePackageReferencesWithSharedReferences() {
CloudRunnerLogger.log(`Use Shared Pkgs ${CloudRunner.buildParameters.useLargePackages}`);
GitHub.updateGitHubCheck(`Use Shared Pkgs ${CloudRunner.buildParameters.useLargePackages}`, ``);
if (CloudRunner.buildParameters.useLargePackages) {
const filePath = path.join(CloudRunnerFolders.projectPathAbsolute, `Packages/manifest.json`);
let manifest = fs.readFileSync(filePath, 'utf8');
manifest = manifest.replace(/LargeContent/g, '../../../LargeContent');
fs.writeFileSync(filePath, manifest);
if (CloudRunner.buildParameters.cloudRunnerDebug) {
CloudRunnerLogger.log(`Package Manifest`);
CloudRunnerLogger.log(manifest);
}
CloudRunnerLogger.log(`Package Manifest \n ${manifest}`);
GitHub.updateGitHubCheck(`Package Manifest \n ${manifest}`, ``);
}
}
@ -121,41 +155,31 @@ export class RemoteClient {
process.chdir(CloudRunnerFolders.repoPathAbsolute);
await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge -- %f"`);
await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process"`);
await CloudRunnerSystem.Run(`git lfs pull`);
RemoteClientLogger.log(`pulled latest LFS files`);
assert(fs.existsSync(CloudRunnerFolders.lfsFolderAbsolute));
}
@CliFunction(`remote-cli-pre-build`, `sets up a repository, usually before a game-ci build`)
static async runRemoteClientJob() {
// await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
RemoteClient.handleRetainedWorkspace();
// await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
await RemoteClient.bootstrapRepository();
// await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
await RemoteClient.runCustomHookFiles(`before-build`);
// await CloudRunnerSystem.Run(`tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
}
static async runCustomHookFiles(hookLifecycle: string) {
RemoteClientLogger.log(`RunCustomHookFiles: ${hookLifecycle}`);
const gameCiCustomHooksPath = path.join(CloudRunnerFolders.repoPathAbsolute, `game-ci`, `hooks`);
const files = fs.readdirSync(gameCiCustomHooksPath);
for (const file of files) {
const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
const fileContentsObject = YAML.parse(fileContents.toString());
if (fileContentsObject.hook === hookLifecycle) {
RemoteClientLogger.log(`Active Hook File ${file} \n \n file contents: \n ${fileContents}`);
await CloudRunnerSystem.Run(fileContentsObject.commands);
}
if (!CloudRunner.buildParameters.skipLfs) {
await CloudRunnerSystem.Run(`git lfs pull`);
RemoteClientLogger.log(`pulled latest LFS files`);
assert(fs.existsSync(CloudRunnerFolders.lfsFolderAbsolute));
}
}
static handleRetainedWorkspace() {
if (!CloudRunner.buildParameters.retainWorkspace) {
return;
static async handleRetainedWorkspace() {
RemoteClientLogger.log(
`Retained Workspace: ${BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters)}`,
);
if (
BuildParameters.shouldUseRetainedWorkspaceMode(CloudRunner.buildParameters) &&
fs.existsSync(CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute)) &&
fs.existsSync(CloudRunnerFolders.ToLinuxFolder(path.join(CloudRunnerFolders.repoPathAbsolute, `.git`)))
) {
CloudRunnerLogger.log(`Retained Workspace Already Exists!`);
process.chdir(CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute));
await CloudRunnerSystem.Run(`git fetch`);
await CloudRunnerSystem.Run(`git lfs pull`);
await CloudRunnerSystem.Run(`git reset --hard "${CloudRunner.buildParameters.gitSha}"`);
await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.gitSha}`);
return true;
}
RemoteClientLogger.log(`Retained Workspace: ${CloudRunner.lockedWorkspace}`);
return false;
}
}

2
src/model/cloud-runner/remote-client/remote-client-logger.ts
View File

@ -1,4 +1,4 @@
import CloudRunnerLogger from '../services/cloud-runner-logger';
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
export class RemoteClientLogger {
public static log(message: string) {

114
src/model/cloud-runner/services/cloud-runner-custom-hooks.ts
View File

@ -1,114 +0,0 @@
import { BuildParameters, Input } from '../..';
import YAML from 'yaml';
import CloudRunnerSecret from './cloud-runner-secret';
import { RemoteClientLogger } from '../remote-client/remote-client-logger';
import path from 'node:path';
import CloudRunnerOptions from '../cloud-runner-options';
import fs from 'node:fs';
// import CloudRunnerLogger from './cloud-runner-logger';
export class CloudRunnerCustomHooks {
// TODO also accept hooks as yaml files in the repo
public static ApplyHooksToCommands(commands: string, buildParameters: BuildParameters): string {
const hooks = CloudRunnerCustomHooks.getHooks(buildParameters.customJobHooks).filter((x) => x.step.includes(`all`));
return `echo "---"
echo "start cloud runner init"
${CloudRunnerOptions.cloudRunnerDebugEnv ? `printenv` : `#`}
echo "start of cloud runner job"
${hooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
${commands}
${hooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
echo "end of cloud runner job"
echo "---${buildParameters.logId}"`;
}
public static getHooks(customJobHooks: string): Hook[] {
const experimentHooks = customJobHooks;
let output = new Array<Hook>();
if (experimentHooks && experimentHooks !== '') {
output = YAML.parse(experimentHooks);
}
return output.filter((x) => x.step !== undefined && x.hook !== undefined && x.hook.length > 0);
}
static GetCustomHooksFromFiles(hookLifecycle: string): Hook[] {
const results: Hook[] = [];
RemoteClientLogger.log(`GetCustomStepFiles: ${hookLifecycle}`);
try {
const gameCiCustomStepsPath = path.join(process.cwd(), `game-ci`, `hooks`);
const files = fs.readdirSync(gameCiCustomStepsPath);
for (const file of files) {
if (!CloudRunnerOptions.customHookFiles.includes(file.replace(`.yaml`, ``))) {
continue;
}
const fileContents = fs.readFileSync(path.join(gameCiCustomStepsPath, file), `utf8`);
const fileContentsObject = CloudRunnerCustomHooks.ParseHooks(fileContents)[0];
if (fileContentsObject.hook.includes(hookLifecycle)) {
results.push(fileContentsObject);
}
}
} catch (error) {
RemoteClientLogger.log(`Failed Getting: ${hookLifecycle} \n ${JSON.stringify(error, undefined, 4)}`);
}
RemoteClientLogger.log(`Active Steps From Files: \n ${JSON.stringify(results, undefined, 4)}`);
return results;
}
private static ConvertYamlSecrets(object: Hook) {
if (object.secrets === undefined) {
object.secrets = [];
return;
}
object.secrets = object.secrets.map((x) => {
return {
ParameterKey: x.ParameterKey,
EnvironmentVariable: Input.ToEnvVarFormat(x.ParameterKey),
ParameterValue: x.ParameterValue,
};
});
}
public static ParseHooks(steps: string): Hook[] {
if (steps === '') {
return [];
}
// if (CloudRunner.buildParameters?.cloudRunnerIntegrationTests) {
// CloudRunnerLogger.log(`Parsing build hooks: ${steps}`);
// }
const isArray = steps.replace(/\s/g, ``)[0] === `-`;
const object: Hook[] = isArray ? YAML.parse(steps) : [YAML.parse(steps)];
for (const hook of object) {
CloudRunnerCustomHooks.ConvertYamlSecrets(hook);
if (hook.secrets === undefined) {
hook.secrets = [];
}
}
if (object === undefined) {
throw new Error(`Failed to parse ${steps}`);
}
return object;
}
public static getSecrets(hooks: Hook[]) {
const secrets = hooks.map((x) => x.secrets).filter((x) => x !== undefined && x.length > 0);
// eslint-disable-next-line unicorn/no-array-reduce
return secrets.length > 0 ? secrets.reduce((x, y) => [...x, ...y]) : [];
}
}
export class Hook {
public commands!: string[];
public secrets: CloudRunnerSecret[] = new Array<CloudRunnerSecret>();
public name!: string;
public hook!: string[];
public step!: string[];
}

0
src/model/cloud-runner/services/cloud-runner-logger.ts → src/model/cloud-runner/services/core/cloud-runner-logger.ts
View File

2
src/model/cloud-runner/services/cloud-runner-system.ts → src/model/cloud-runner/services/core/cloud-runner-system.ts
View File

@ -1,5 +1,5 @@
import { exec } from 'child_process';
import { RemoteClientLogger } from '../remote-client/remote-client-logger';
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
export class CloudRunnerSystem {
public static async RunAndReadLines(command: string): Promise<string[]> {

57
src/model/cloud-runner/services/core/follow-log-stream-service.ts Normal file
View File

@ -0,0 +1,57 @@
import GitHub from '../../../github';
import CloudRunner from '../../cloud-runner';
import { CloudRunnerStatics } from '../../options/cloud-runner-statics';
import CloudRunnerLogger from './cloud-runner-logger';
import * as core from '@actions/core';
export class FollowLogStreamService {
static Reset() {
FollowLogStreamService.DidReceiveEndOfTransmission = false;
}
static errors = ``;
public static DidReceiveEndOfTransmission = false;
public static handleIteration(message: string, shouldReadLogs: boolean, shouldCleanup: boolean, output: string) {
if (message.includes(`---${CloudRunner.buildParameters.logId}`)) {
CloudRunnerLogger.log('End of log transmission received');
FollowLogStreamService.DidReceiveEndOfTransmission = true;
shouldReadLogs = false;
} else if (message.includes('Rebuilding Library because the asset database could not be found!')) {
GitHub.updateGitHubCheck(`Library was not found, importing new Library`, ``);
core.warning('LIBRARY NOT FOUND!');
core.setOutput('library-found', 'false');
} else if (message.includes('Build succeeded')) {
GitHub.updateGitHubCheck(`Build succeeded`, `Build succeeded`);
core.setOutput('build-result', 'success');
} else if (message.includes('Build fail')) {
GitHub.updateGitHubCheck(
`Build failed\n${FollowLogStreamService.errors}`,
`Build failed`,
`failure`,
`completed`,
);
core.setOutput('build-result', 'failed');
core.setFailed('unity build failed');
core.error('BUILD FAILED!');
} else if (message.toLowerCase().includes('error ')) {
core.error(message);
FollowLogStreamService.errors += `\n${message}`;
} else if (message.toLowerCase().includes('error: ')) {
core.error(message);
FollowLogStreamService.errors += `\n${message}`;
} else if (message.toLowerCase().includes('command failed: ')) {
FollowLogStreamService.errors += `\n${message}`;
} else if (message.toLowerCase().includes('invalid ')) {
FollowLogStreamService.errors += `\n${message}`;
} else if (message.toLowerCase().includes('incompatible ')) {
FollowLogStreamService.errors += `\n${message}`;
} else if (message.toLowerCase().includes('cannot be found')) {
FollowLogStreamService.errors += `\n${message}`;
}
if (CloudRunner.buildParameters.cloudRunnerDebug) {
output += `${message}\n`;
}
CloudRunnerLogger.log(`[${CloudRunnerStatics.logPrefix}] ${message}`);
return { shouldReadLogs, shouldCleanup, output };
}
}

189
src/model/cloud-runner/services/shared-workspace-locking.ts → src/model/cloud-runner/services/core/shared-workspace-locking.ts
View File

@ -1,18 +1,17 @@
import { CloudRunnerSystem } from './cloud-runner-system';
import fs from 'node:fs';
import CloudRunnerLogger from './cloud-runner-logger';
import CloudRunnerOptions from '../cloud-runner-options';
import BuildParameters from '../../build-parameters';
import CloudRunner from '../cloud-runner';
import BuildParameters from '../../../build-parameters';
import CloudRunner from '../../cloud-runner';
export class SharedWorkspaceLocking {
private static get workspaceBucketRoot() {
return `s3://${CloudRunner.buildParameters.awsBaseStackName}/`;
public static get workspaceBucketRoot() {
return `s3://${CloudRunner.buildParameters.awsStackName}/`;
}
private static get workspaceRoot() {
public static get workspaceRoot() {
return `${SharedWorkspaceLocking.workspaceBucketRoot}locks/`;
}
public static async GetAllWorkspaces(buildParametersContext: BuildParameters): Promise<string[]> {
if (!(await SharedWorkspaceLocking.DoesWorkspaceTopLevelExist(buildParametersContext))) {
if (!(await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParametersContext))) {
return [];
}
@ -20,79 +19,95 @@ export class SharedWorkspaceLocking {
await SharedWorkspaceLocking.ReadLines(
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
)
).map((x) => x.replace(`/`, ``));
}
public static async DoesWorkspaceTopLevelExist(buildParametersContext: BuildParameters) {
await SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceBucketRoot}`);
return (await SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}`))
)
.map((x) => x.replace(`/`, ``))
.includes(buildParametersContext.cacheKey);
.filter((x) => x.endsWith(`_workspace`))
.map((x) => x.split(`_`)[1]);
}
public static async GetAllLocks(workspace: string, buildParametersContext: BuildParameters): Promise<string[]> {
public static async DoesCacheKeyTopLevelExist(buildParametersContext: BuildParameters) {
try {
const rootLines = await SharedWorkspaceLocking.ReadLines(
`aws s3 ls ${SharedWorkspaceLocking.workspaceBucketRoot}`,
);
const lockFolderExists = rootLines.map((x) => x.replace(`/`, ``)).includes(`locks`);
if (lockFolderExists) {
const lines = await SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}`);
return lines.map((x) => x.replace(`/`, ``)).includes(buildParametersContext.cacheKey);
} else {
return false;
}
} catch {
return false;
}
}
public static NewWorkspaceName() {
return `${CloudRunner.retainedWorkspacePrefix}-${CloudRunner.buildParameters.buildGuid}`;
}
public static async GetAllLocksForWorkspace(
workspace: string,
buildParametersContext: BuildParameters,
): Promise<string[]> {
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
return [];
}
return (
await SharedWorkspaceLocking.ReadLines(
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/`,
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
)
)
.map((x) => x.replace(`/`, ``))
.filter((x) => x.includes(`_lock`));
.filter((x) => x.includes(workspace) && x.endsWith(`_lock`));
}
public static async GetOrCreateLockedWorkspace(
workspace: string,
runId: string,
buildParametersContext: BuildParameters,
) {
if (!CloudRunnerOptions.retainWorkspaces) {
return;
public static async GetLockedWorkspace(workspace: string, runId: string, buildParametersContext: BuildParameters) {
if (buildParametersContext.maxRetainedWorkspaces === 0) {
return false;
}
try {
if (await SharedWorkspaceLocking.DoesWorkspaceTopLevelExist(buildParametersContext)) {
const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces(buildParametersContext);
if (await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParametersContext)) {
const workspaces = await SharedWorkspaceLocking.GetFreeWorkspaces(buildParametersContext);
CloudRunnerLogger.log(`run agent ${runId} is trying to access a workspace, free: ${JSON.stringify(workspaces)}`);
for (const element of workspaces) {
const lockResult = await SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext);
CloudRunnerLogger.log(
`run agent ${runId} is trying to access a workspace, free: ${JSON.stringify(workspaces)}`,
`run agent: ${runId} try lock workspace: ${element} locking attempt result: ${lockResult}`,
);
for (const element of workspaces) {
await new Promise((promise) => setTimeout(promise, 1000));
const lockResult = await SharedWorkspaceLocking.LockWorkspace(element, runId, buildParametersContext);
CloudRunnerLogger.log(`run agent: ${runId} try lock workspace: ${element} result: ${lockResult}`);
if (lockResult) {
CloudRunner.lockedWorkspace = element;
return true;
}
if (lockResult) {
return true;
}
}
} catch {
return;
}
const createResult = await SharedWorkspaceLocking.CreateWorkspace(workspace, buildParametersContext, runId);
if (await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext)) {
workspace = SharedWorkspaceLocking.NewWorkspaceName();
CloudRunner.lockedWorkspace = workspace;
}
const createResult = await SharedWorkspaceLocking.CreateWorkspace(workspace, buildParametersContext);
const lockResult = await SharedWorkspaceLocking.LockWorkspace(workspace, runId, buildParametersContext);
CloudRunnerLogger.log(
`run agent ${runId} didn't find a free workspace so created: ${workspace} createWorkspaceSuccess: ${createResult}`,
`run agent ${runId} didn't find a free workspace so created: ${workspace} createWorkspaceSuccess: ${createResult} Lock:${lockResult}`,
);
return createResult;
return createResult && lockResult;
}
public static async DoesWorkspaceExist(workspace: string, buildParametersContext: BuildParameters) {
return (await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext)).includes(workspace);
return (
(await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext)).filter((x) => x.includes(workspace))
.length > 0
);
}
public static async HasWorkspaceLock(
workspace: string,
runId: string,
buildParametersContext: BuildParameters,
): Promise<boolean> {
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
return false;
}
const locks = (await SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext))
const locks = (await SharedWorkspaceLocking.GetAllLocksForWorkspace(workspace, buildParametersContext))
.map((x) => {
return {
name: x,
@ -115,14 +130,11 @@ export class SharedWorkspaceLocking {
const result: string[] = [];
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
for (const element of workspaces) {
await new Promise((promise) => setTimeout(promise, 1500));
const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParametersContext);
const isBelowMax = await SharedWorkspaceLocking.IsWorkspaceBelowMax(element, buildParametersContext);
CloudRunnerLogger.log(`workspace ${element} locked:${isLocked} below max:${isBelowMax}`);
if (!isLocked && isBelowMax) {
result.push(element);
CloudRunnerLogger.log(`workspace ${element} is free`);
} else {
CloudRunnerLogger.log(`workspace ${element} is NOT free ${!isLocked} ${isBelowMax}`);
}
}
@ -171,60 +183,49 @@ export class SharedWorkspaceLocking {
return (
await SharedWorkspaceLocking.ReadLines(
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/`,
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
)
)
.map((x) => x.replace(`/`, ``))
.filter((x) => x.includes(`_workspace`))
.filter((x) => x.includes(workspace) && x.endsWith(`_workspace`))
.map((x) => Number(x))[0];
}
public static async IsWorkspaceLocked(workspace: string, buildParametersContext: BuildParameters): Promise<boolean> {
if (!(await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext))) {
return false;
throw new Error(`workspace doesn't exist ${workspace}`);
}
const files = await SharedWorkspaceLocking.ReadLines(
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/`,
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
);
const workspaceFileDoesNotExists =
files.filter((x) => {
return x.includes(`_workspace`);
}).length === 0;
const lockFilesExist =
files.filter((x) => {
return x.includes(`_lock`);
return x.includes(workspace) && x.endsWith(`_lock`);
}).length > 0;
return workspaceFileDoesNotExists || lockFilesExist;
return lockFilesExist;
}
public static async CreateWorkspace(
workspace: string,
buildParametersContext: BuildParameters,
lockId: string = ``,
): Promise<boolean> {
if (lockId !== ``) {
await SharedWorkspaceLocking.LockWorkspace(workspace, lockId, buildParametersContext);
public static async CreateWorkspace(workspace: string, buildParametersContext: BuildParameters): Promise<boolean> {
if (await SharedWorkspaceLocking.DoesWorkspaceExist(workspace, buildParametersContext)) {
throw new Error(`${workspace} already exists`);
}
const timestamp = Date.now();
const file = `${timestamp}_workspace`;
const file = `${timestamp}_${workspace}_workspace`;
fs.writeFileSync(file, '');
await CloudRunnerSystem.Run(
`aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`,
`aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`,
false,
true,
);
fs.rmSync(file);
const workspaces = await SharedWorkspaceLocking.ReadLines(
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/`,
);
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParametersContext);
CloudRunnerLogger.log(`All workspaces ${workspaces}`);
if (!(await SharedWorkspaceLocking.IsWorkspaceBelowMax(workspace, buildParametersContext))) {
CloudRunnerLogger.log(`Workspace is below max ${workspaces} ${buildParametersContext.maxRetainedWorkspaces}`);
CloudRunnerLogger.log(`Workspace is above max ${workspaces} ${buildParametersContext.maxRetainedWorkspaces}`);
await SharedWorkspaceLocking.CleanupWorkspace(workspace, buildParametersContext);
return false;
@ -238,16 +239,30 @@ export class SharedWorkspaceLocking {
runId: string,
buildParametersContext: BuildParameters,
): Promise<boolean> {
const file = `${Date.now()}_${runId}_lock`;
const existingWorkspace = workspace.endsWith(`_workspace`);
const ending = existingWorkspace ? workspace : `${workspace}_workspace`;
const file = `${Date.now()}_${runId}_${ending}_lock`;
fs.writeFileSync(file, '');
await CloudRunnerSystem.Run(
`aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`,
`aws s3 cp ./${file} ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`,
false,
true,
);
fs.rmSync(file);
return SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext);
const hasLock = await SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext);
if (hasLock) {
CloudRunner.lockedWorkspace = workspace;
} else {
await CloudRunnerSystem.Run(
`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`,
false,
true,
);
}
return hasLock;
}
public static async ReleaseWorkspace(
@ -255,31 +270,29 @@ export class SharedWorkspaceLocking {
runId: string,
buildParametersContext: BuildParameters,
): Promise<boolean> {
const file = (await SharedWorkspaceLocking.GetAllLocks(workspace, buildParametersContext)).filter((x) =>
x.includes(`_${runId}_lock`),
);
const files = await SharedWorkspaceLocking.GetAllLocksForWorkspace(workspace, buildParametersContext);
const file = files.find((x) => x.includes(workspace) && x.endsWith(`_lock`) && x.includes(runId));
CloudRunnerLogger.log(`All Locks ${files} ${workspace} ${runId}`);
CloudRunnerLogger.log(`Deleting lock ${workspace}/${file}`);
CloudRunnerLogger.log(
`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`,
);
CloudRunnerLogger.log(`rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`);
await CloudRunnerSystem.Run(
`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace}/${file}`,
`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${file}`,
false,
true,
);
return !SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext);
return !(await SharedWorkspaceLocking.HasWorkspaceLock(workspace, runId, buildParametersContext));
}
public static async CleanupWorkspace(workspace: string, buildParametersContext: BuildParameters) {
await CloudRunnerSystem.Run(
`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey}/${workspace} --recursive`,
`aws s3 rm ${SharedWorkspaceLocking.workspaceRoot}${buildParametersContext.cacheKey} --exclude "*" --include "*_${workspace}_*"`,
false,
true,
);
}
private static async ReadLines(command: string): Promise<string[]> {
public static async ReadLines(command: string): Promise<string[]> {
return CloudRunnerSystem.RunAndReadLines(command);
}
}

107
src/model/cloud-runner/services/task-parameter-serializer.ts → src/model/cloud-runner/services/core/task-parameter-serializer.ts
View File

@ -1,53 +1,50 @@
import { Input } from '../..';
import CloudRunnerEnvironmentVariable from './cloud-runner-environment-variable';
import { CloudRunnerCustomHooks } from './cloud-runner-custom-hooks';
import CloudRunnerSecret from './cloud-runner-secret';
import CloudRunnerQueryOverride from './cloud-runner-query-override';
import CloudRunnerOptionsReader from './cloud-runner-options-reader';
import BuildParameters from '../../build-parameters';
import CloudRunnerOptions from '../cloud-runner-options';
import * as core from '@actions/core';
import BuildParameters from '../../../build-parameters';
import Input from '../../../input';
import CloudRunnerOptions from '../../options/cloud-runner-options';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import CloudRunnerOptionsReader from '../../options/cloud-runner-options-reader';
import CloudRunnerQueryOverride from '../../options/cloud-runner-query-override';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import { CommandHookService } from '../hooks/command-hook-service';
export class TaskParameterSerializer {
static readonly blocked = new Set(['0', 'length', 'prototype', '', 'unityVersion']);
static readonly blockedParameterNames: Set<string> = new Set([
'0',
'length',
'prototype',
'',
'unityVersion',
'CACHE_UNITY_INSTALLATION_ON_MAC',
'RUNNER_TEMP_PATH',
'NAME',
'CUSTOM_JOB',
]);
public static createCloudRunnerEnvironmentVariables(
buildParameters: BuildParameters,
): CloudRunnerEnvironmentVariable[] {
const result = this.uniqBy(
const result: CloudRunnerEnvironmentVariable[] = this.uniqBy(
[
{
name: 'ContainerMemory',
value: buildParameters.cloudRunnerMemory,
},
{
name: 'ContainerCpu',
value: buildParameters.cloudRunnerCpu,
},
{
name: 'BUILD_TARGET',
value: buildParameters.targetPlatform,
},
...[
{ name: 'BUILD_TARGET', value: buildParameters.targetPlatform },
{ name: 'UNITY_VERSION', value: buildParameters.editorVersion },
{ name: 'GITHUB_TOKEN', value: process.env.GITHUB_TOKEN },
],
...TaskParameterSerializer.serializeFromObject(buildParameters),
...TaskParameterSerializer.readInput(),
...CloudRunnerCustomHooks.getSecrets(CloudRunnerCustomHooks.getHooks(buildParameters.customJobHooks)),
...TaskParameterSerializer.serializeInput(),
...TaskParameterSerializer.serializeCloudRunnerOptions(),
...CommandHookService.getSecrets(CommandHookService.getHooks(buildParameters.commandHooks)),
]
.filter(
(x) =>
!TaskParameterSerializer.blocked.has(x.name) &&
!TaskParameterSerializer.blockedParameterNames.has(x.name) &&
x.value !== '' &&
x.value !== undefined &&
x.name !== `CUSTOM_JOB` &&
x.name !== `GAMECI_CUSTOM_JOB` &&
x.value !== `undefined`,
)
.map((x) => {
x.name = TaskParameterSerializer.ToEnvVarFormat(x.name);
x.name = `${TaskParameterSerializer.ToEnvVarFormat(x.name)}`;
x.value = `${x.value}`;
if (buildParameters.cloudRunnerDebug && Number.isNaN(Number(x.name))) {
core.info(`[ERROR] found a number in task param serializer ${JSON.stringify(x)}`);
}
return x;
}),
(item: CloudRunnerEnvironmentVariable) => item.name,
@ -72,54 +69,58 @@ export class TaskParameterSerializer {
const keys = [
...new Set(
Object.getOwnPropertyNames(process.env)
.filter((x) => !this.blocked.has(x) && x.startsWith('GAMECI_'))
.filter((x) => !this.blockedParameterNames.has(x) && x.startsWith(''))
.map((x) => TaskParameterSerializer.UndoEnvVarFormat(x)),
),
];
for (const element of keys) {
if (element !== `customJob`) {
buildParameters[element] = process.env[`GAMECI_${TaskParameterSerializer.ToEnvVarFormat(element)}`];
buildParameters[element] = process.env[`${TaskParameterSerializer.ToEnvVarFormat(element)}`];
}
}
return buildParameters;
}
private static readInput() {
private static serializeInput() {
return TaskParameterSerializer.serializeFromType(Input);
}
private static serializeCloudRunnerOptions() {
return TaskParameterSerializer.serializeFromType(CloudRunnerOptions);
}
public static ToEnvVarFormat(input: string): string {
return CloudRunnerOptions.ToEnvVarFormat(input);
}
public static UndoEnvVarFormat(element: string): string {
return this.camelize(element.replace('GAMECI_', '').toLowerCase().replace(/_+/g, ' '));
return this.camelize(element.toLowerCase().replace(/_+/g, ' '));
}
private static camelize(string: string) {
return string
.replace(/(^\w)|([A-Z])|(\b\w)/g, function (word: string, index: number) {
return index === 0 ? word.toLowerCase() : word.toUpperCase();
})
.replace(/\s+/g, '');
return TaskParameterSerializer.uncapitalizeFirstLetter(
string
.replace(/(^\w)|([A-Z])|(\b\w)/g, function (word: string, index: number) {
return index === 0 ? word.toLowerCase() : word.toUpperCase();
})
.replace(/\s+/g, ''),
);
}
private static uncapitalizeFirstLetter(string: string) {
return string.charAt(0).toLowerCase() + string.slice(1);
}
private static serializeFromObject(buildParameters: any) {
const array: any[] = [];
const keys = Object.getOwnPropertyNames(buildParameters).filter((x) => !this.blocked.has(x));
const keys = Object.getOwnPropertyNames(buildParameters).filter((x) => !this.blockedParameterNames.has(x));
for (const element of keys) {
array.push(
{
name: `GAMECI_${TaskParameterSerializer.ToEnvVarFormat(element)}`,
value: buildParameters[element],
},
{
name: element,
value: buildParameters[element],
},
);
array.push({
name: TaskParameterSerializer.ToEnvVarFormat(element),
value: buildParameters[element],
});
}
return array;

37
src/model/cloud-runner/services/follow-log-stream-service.ts
View File

@ -1,37 +0,0 @@
import CloudRunnerLogger from './cloud-runner-logger';
import * as core from '@actions/core';
import CloudRunner from '../cloud-runner';
import { CloudRunnerStatics } from '../cloud-runner-statics';
import GitHub from '../../github';
export class FollowLogStreamService {
public static handleIteration(message: string, shouldReadLogs: boolean, shouldCleanup: boolean, output: string) {
if (message.includes(`---${CloudRunner.buildParameters.logId}`)) {
CloudRunnerLogger.log('End of log transmission received');
shouldReadLogs = false;
} else if (message.includes('Rebuilding Library because the asset database could not be found!')) {
GitHub.updateGitHubCheck(`Library was not found, importing new Library`, ``);
core.warning('LIBRARY NOT FOUND!');
core.setOutput('library-found', 'false');
} else if (message.includes('Build succeeded')) {
GitHub.updateGitHubCheck(`Build succeeded`, `Build succeeded`);
core.setOutput('build-result', 'success');
} else if (message.includes('Build fail')) {
GitHub.updateGitHubCheck(`Build failed`, `Build failed`);
core.setOutput('build-result', 'failed');
core.setFailed('unity build failed');
core.error('BUILD FAILED!');
} else if (CloudRunner.buildParameters.cloudRunnerDebug && message.includes(': Listening for Jobs')) {
core.setOutput('cloud runner stop watching', 'true');
shouldReadLogs = false;
shouldCleanup = false;
core.warning('cloud runner stop watching');
}
if (CloudRunner.buildParameters.cloudRunnerDebug) {
output += `${message}\n`;
}
CloudRunnerLogger.log(`[${CloudRunnerStatics.logPrefix}] ${message}`);
return { shouldReadLogs, shouldCleanup, output };
}
}

118
src/model/cloud-runner/services/hooks/command-hook-service.ts Normal file
View File

@ -0,0 +1,118 @@
import { BuildParameters, Input } from '../../..';
import YAML from 'yaml';
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
import path from 'node:path';
import CloudRunnerOptions from '../../options/cloud-runner-options';
import * as fs from 'node:fs';
import CloudRunnerLogger from '../core/cloud-runner-logger';
import { CommandHook } from './command-hook';
// import CloudRunnerLogger from './cloud-runner-logger';
export class CommandHookService {
public static ApplyHooksToCommands(commands: string, buildParameters: BuildParameters): string {
const hooks = CommandHookService.getHooks(buildParameters.commandHooks);
CloudRunnerLogger.log(`Applying hooks ${hooks.length}`);
return `echo "---"
echo "start cloud runner init"
${CloudRunnerOptions.cloudRunnerDebug ? `printenv` : `#`}
echo "start of cloud runner job"
${hooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
${commands}
${hooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
echo "end of cloud runner job"
echo "---${buildParameters.logId}"`;
}
public static getHooks(customCommandHooks: string): CommandHook[] {
const experimentHooks = customCommandHooks;
let output = new Array<CommandHook>();
if (experimentHooks && experimentHooks !== '') {
try {
output = YAML.parse(experimentHooks);
} catch (error) {
throw error;
}
}
return [
...output.filter((x) => x.hook !== undefined && x.hook.length > 0),
...CommandHookService.GetCustomHooksFromFiles(`before`),
...CommandHookService.GetCustomHooksFromFiles(`after`),
];
}
static GetCustomHooksFromFiles(hookLifecycle: string): CommandHook[] {
const results: CommandHook[] = [];
// RemoteClientLogger.log(`GetCustomHookFiles: ${hookLifecycle}`);
try {
const gameCiCustomHooksPath = path.join(process.cwd(), `game-ci`, `command-hooks`);
const files = fs.readdirSync(gameCiCustomHooksPath);
for (const file of files) {
if (!CloudRunnerOptions.commandHookFiles.includes(file.replace(`.yaml`, ``))) {
continue;
}
const fileContents = fs.readFileSync(path.join(gameCiCustomHooksPath, file), `utf8`);
const fileContentsObject = CommandHookService.ParseHooks(fileContents)[0];
if (fileContentsObject.hook.includes(hookLifecycle)) {
results.push(fileContentsObject);
}
}
} catch (error) {
RemoteClientLogger.log(`Failed Getting: ${hookLifecycle} \n ${JSON.stringify(error, undefined, 4)}`);
}
// RemoteClientLogger.log(`Active Steps From Hooks: \n ${JSON.stringify(results, undefined, 4)}`);
return results;
}
private static ConvertYamlSecrets(object: CommandHook) {
if (object.secrets === undefined) {
object.secrets = [];
return;
}
object.secrets = object.secrets.map((x: any) => {
return {
ParameterKey: x.name,
EnvironmentVariable: Input.ToEnvVarFormat(x.name),
ParameterValue: x.value,
};
});
}
public static ParseHooks(hooks: string): CommandHook[] {
if (hooks === '') {
return [];
}
// if (CloudRunner.buildParameters?.cloudRunnerIntegrationTests) {
// CloudRunnerLogger.log(`Parsing build hooks: ${steps}`);
// }
const isArray = hooks.replace(/\s/g, ``)[0] === `-`;
const object: CommandHook[] = isArray ? YAML.parse(hooks) : [YAML.parse(hooks)];
for (const hook of object) {
CommandHookService.ConvertYamlSecrets(hook);
if (hook.secrets === undefined) {
hook.secrets = [];
}
}
if (object === undefined) {
throw new Error(`Failed to parse ${hooks}`);
}
return object;
}
public static getSecrets(hooks: any) {
const secrets = hooks.map((x: any) => x.secrets).filter((x: any) => x !== undefined && x.length > 0);
// eslint-disable-next-line unicorn/no-array-reduce
return secrets.length > 0 ? secrets.reduce((x: any, y: any) => [...x, ...y]) : [];
}
}

9
src/model/cloud-runner/services/hooks/command-hook.ts Normal file
View File

@ -0,0 +1,9 @@
import CloudRunnerSecret from '../../options/cloud-runner-secret';
export class CommandHook {
public commands: string[] = new Array<string>();
public secrets: CloudRunnerSecret[] = new Array<CloudRunnerSecret>();
public name!: string;
public hook!: string[];
public step!: string[];
}

128
src/model/cloud-runner/services/cloud-runner-custom-steps.ts → src/model/cloud-runner/services/hooks/container-hook-service.ts
View File

@ -1,32 +1,28 @@
import YAML from 'yaml';
import CloudRunner from '../cloud-runner';
import CloudRunner from '../../cloud-runner';
import * as core from '@actions/core';
import { CustomWorkflow } from '../workflows/custom-workflow';
import { RemoteClientLogger } from '../remote-client/remote-client-logger';
import { CustomWorkflow } from '../../workflows/custom-workflow';
import { RemoteClientLogger } from '../../remote-client/remote-client-logger';
import path from 'node:path';
import fs from 'node:fs';
import Input from '../../input';
import CloudRunnerOptions from '../cloud-runner-options';
import CloudRunnerLogger from './cloud-runner-logger';
import { CustomStep } from './custom-step';
import { CloudRunnerStepState } from '../cloud-runner-step-state';
import Input from '../../../input';
import CloudRunnerOptions from '../../options/cloud-runner-options';
import { ContainerHook as ContainerHook } from './container-hook';
import { CloudRunnerStepParameters } from '../../options/cloud-runner-step-parameters';
export class CloudRunnerCustomSteps {
static GetCustomStepsFromFiles(hookLifecycle: string): CustomStep[] {
const results: CustomStep[] = [];
RemoteClientLogger.log(
`GetCustomStepFiles: ${hookLifecycle} CustomStepFiles: ${CloudRunnerOptions.customStepFiles}`,
);
export class ContainerHookService {
static GetContainerHooksFromFiles(hookLifecycle: string): ContainerHook[] {
const results: ContainerHook[] = [];
try {
const gameCiCustomStepsPath = path.join(process.cwd(), `game-ci`, `steps`);
const gameCiCustomStepsPath = path.join(process.cwd(), `game-ci`, `container-hooks`);
const files = fs.readdirSync(gameCiCustomStepsPath);
for (const file of files) {
if (!CloudRunnerOptions.customStepFiles.includes(file.replace(`.yaml`, ``))) {
RemoteClientLogger.log(`Skipping CustomStepFile: ${file}`);
if (!CloudRunnerOptions.containerHookFiles.includes(file.replace(`.yaml`, ``))) {
// RemoteClientLogger.log(`Skipping CustomStepFile: ${file}`);
continue;
}
const fileContents = fs.readFileSync(path.join(gameCiCustomStepsPath, file), `utf8`);
const fileContentsObject = CloudRunnerCustomSteps.ParseSteps(fileContents)[0];
const fileContentsObject = ContainerHookService.ParseContainerHooks(fileContents)[0];
if (fileContentsObject.hook === hookLifecycle) {
results.push(fileContentsObject);
}
@ -34,9 +30,10 @@ export class CloudRunnerCustomSteps {
} catch (error) {
RemoteClientLogger.log(`Failed Getting: ${hookLifecycle} \n ${JSON.stringify(error, undefined, 4)}`);
}
RemoteClientLogger.log(`Active Steps From Files: \n ${JSON.stringify(results, undefined, 4)}`);
const builtInCustomSteps: CustomStep[] = CloudRunnerCustomSteps.ParseSteps(
// RemoteClientLogger.log(`Active Steps From Files: \n ${JSON.stringify(results, undefined, 4)}`);
const builtInContainerHooks: ContainerHook[] = ContainerHookService.ParseContainerHooks(
`- name: aws-s3-upload-build
image: amazon/aws-cli
hook: after
@ -45,12 +42,12 @@ export class CloudRunnerCustomSteps {
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
aws configure set region $AWS_DEFAULT_REGION --profile default
aws s3 cp /data/cache/$CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
} s3://${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID.tar${
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
} s3://${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
}
rm /data/cache/$CACHE_KEY/build/build-${CloudRunner.buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
}
secrets:
- name: awsAccessKeyId
@ -65,19 +62,20 @@ export class CloudRunnerCustomSteps {
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
aws configure set region $AWS_DEFAULT_REGION --profile default
aws s3 ls ${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/ || true
aws s3 ls ${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/$CACHE_KEY/build || true
aws s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/ || true
aws s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/build || true
mkdir -p /data/cache/$CACHE_KEY/build/
aws s3 cp s3://${
CloudRunner.buildParameters.awsBaseStackName
CloudRunner.buildParameters.awsStackName
}/cloud-runner-cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
} /data/cache/$CACHE_KEY/build/build-$BUILD_GUID_TARGET.tar${
CloudRunner.buildParameters.useLz4Compression ? '.lz4' : ''
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
}
secrets:
- name: awsAccessKeyId
- name: awsSecretAccessKey
- name: awsDefaultRegion
- name: AWS_ACCESS_KEY_ID
- name: AWS_SECRET_ACCESS_KEY
- name: AWS_DEFAULT_REGION
- name: BUILD_GUID_TARGET
- name: steam-deploy-client
image: steamcmd/steamcmd
@ -123,19 +121,19 @@ export class CloudRunnerCustomSteps {
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
aws configure set region $AWS_DEFAULT_REGION --profile default
aws s3 cp --recursive /data/cache/$CACHE_KEY/lfs s3://${
CloudRunner.buildParameters.awsBaseStackName
CloudRunner.buildParameters.awsStackName
}/cloud-runner-cache/$CACHE_KEY/lfs
rm -r /data/cache/$CACHE_KEY/lfs
aws s3 cp --recursive /data/cache/$CACHE_KEY/Library s3://${
CloudRunner.buildParameters.awsBaseStackName
CloudRunner.buildParameters.awsStackName
}/cloud-runner-cache/$CACHE_KEY/Library
rm -r /data/cache/$CACHE_KEY/Library
secrets:
- name: awsAccessKeyId
- name: AWS_ACCESS_KEY_ID
value: ${process.env.AWS_ACCESS_KEY_ID || ``}
- name: awsSecretAccessKey
- name: AWS_SECRET_ACCESS_KEY
value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
- name: awsDefaultRegion
- name: AWS_DEFAULT_REGION
value: ${process.env.AWS_REGION || ``}
- name: aws-s3-pull-cache
image: amazon/aws-cli
@ -144,30 +142,32 @@ export class CloudRunnerCustomSteps {
aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID --profile default
aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY --profile default
aws configure set region $AWS_DEFAULT_REGION --profile default
aws s3 ls ${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/ || true
aws s3 ls ${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/$CACHE_KEY/ || true
BUCKET1="${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/$CACHE_KEY/Library/"
mkdir -p /data/cache/$CACHE_KEY/Library/
mkdir -p /data/cache/$CACHE_KEY/lfs/
aws s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/ || true
aws s3 ls ${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/ || true
BUCKET1="${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/Library/"
aws s3 ls $BUCKET1 || true
OBJECT1="$(aws s3 ls $BUCKET1 | sort | tail -n 1 | awk '{print $4}' || '')"
aws s3 cp s3://$BUCKET1$OBJECT1 /data/cache/$CACHE_KEY/Library/ || true
BUCKET2="${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/$CACHE_KEY/lfs/"
BUCKET2="${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/$CACHE_KEY/lfs/"
aws s3 ls $BUCKET2 || true
OBJECT2="$(aws s3 ls $BUCKET2 | sort | tail -n 1 | awk '{print $4}' || '')"
aws s3 cp s3://$BUCKET2$OBJECT2 /data/cache/$CACHE_KEY/lfs/ || true
secrets:
- name: awsAccessKeyId
- name: AWS_ACCESS_KEY_ID
value: ${process.env.AWS_ACCESS_KEY_ID || ``}
- name: awsSecretAccessKey
- name: AWS_SECRET_ACCESS_KEY
value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
- name: awsDefaultRegion
- name: AWS_DEFAULT_REGION
value: ${process.env.AWS_REGION || ``}
- name: debug-cache
image: ubuntu
hook: after
commands: |
apt-get update > /dev/null
${CloudRunnerOptions.cloudRunnerDebugTree ? `apt-get install -y tree > /dev/null` : `#`}
${CloudRunnerOptions.cloudRunnerDebugTree ? `tree -L 3 /data/cache` : `#`}
${CloudRunnerOptions.cloudRunnerDebug ? `apt-get install -y tree > /dev/null` : `#`}
${CloudRunnerOptions.cloudRunnerDebug ? `tree -L 3 /data/cache` : `#`}
secrets:
- name: awsAccessKeyId
value: ${process.env.AWS_ACCESS_KEY_ID || ``}
@ -175,15 +175,15 @@ export class CloudRunnerCustomSteps {
value: ${process.env.AWS_SECRET_ACCESS_KEY || ``}
- name: awsDefaultRegion
value: ${process.env.AWS_REGION || ``}`,
).filter((x) => CloudRunnerOptions.customStepFiles.includes(x.name) && x.hook === hookLifecycle);
if (builtInCustomSteps.length > 0) {
results.push(...builtInCustomSteps);
).filter((x) => CloudRunnerOptions.containerHookFiles.includes(x.name) && x.hook === hookLifecycle);
if (builtInContainerHooks.length > 0) {
results.push(...builtInContainerHooks);
}
return results;
}
private static ConvertYamlSecrets(object: CustomStep) {
private static ConvertYamlSecrets(object: ContainerHook) {
if (object.secrets === undefined) {
object.secrets = [];
@ -198,21 +198,21 @@ export class CloudRunnerCustomSteps {
});
}
public static ParseSteps(steps: string): CustomStep[] {
public static ParseContainerHooks(steps: string): ContainerHook[] {
if (steps === '') {
return [];
}
const isArray = steps.replace(/\s/g, ``)[0] === `-`;
const object: CustomStep[] = isArray ? YAML.parse(steps) : [YAML.parse(steps)];
const object: ContainerHook[] = isArray ? YAML.parse(steps) : [YAML.parse(steps)];
for (const step of object) {
CloudRunnerCustomSteps.ConvertYamlSecrets(step);
ContainerHookService.ConvertYamlSecrets(step);
if (step.secrets === undefined) {
step.secrets = [];
} else {
for (const secret of step.secrets) {
if (secret.ParameterValue === undefined && process.env[secret.EnvironmentVariable] !== undefined) {
if (CloudRunner.buildParameters?.cloudRunnerDebug) {
CloudRunnerLogger.log(`Injecting custom step ${step.name} from env var ${secret.ParameterKey}`);
// CloudRunnerLogger.log(`Injecting custom step ${step.name} from env var ${secret.ParameterKey}`);
}
secret.ParameterValue = process.env[secret.ParameterKey] || ``;
}
@ -229,16 +229,16 @@ export class CloudRunnerCustomSteps {
return object;
}
static async RunPostBuildSteps(cloudRunnerStepState: CloudRunnerStepState) {
static async RunPostBuildSteps(cloudRunnerStepState: CloudRunnerStepParameters) {
let output = ``;
const steps: CustomStep[] = [
...CloudRunnerCustomSteps.ParseSteps(CloudRunner.buildParameters.postBuildSteps),
...CloudRunnerCustomSteps.GetCustomStepsFromFiles(`after`),
const steps: ContainerHook[] = [
...ContainerHookService.ParseContainerHooks(CloudRunner.buildParameters.postBuildContainerHooks),
...ContainerHookService.GetContainerHooksFromFiles(`after`),
];
if (steps.length > 0) {
if (!CloudRunner.buildParameters.isCliMode) core.startGroup('post build steps');
output += await CustomWorkflow.runCustomJob(
output += await CustomWorkflow.runContainerJob(
steps,
cloudRunnerStepState.environment,
cloudRunnerStepState.secrets,
@ -248,16 +248,16 @@ export class CloudRunnerCustomSteps {
return output;
}
static async RunPreBuildSteps(cloudRunnerStepState: CloudRunnerStepState) {
static async RunPreBuildSteps(cloudRunnerStepState: CloudRunnerStepParameters) {
let output = ``;
const steps: CustomStep[] = [
...CloudRunnerCustomSteps.ParseSteps(CloudRunner.buildParameters.preBuildSteps),
...CloudRunnerCustomSteps.GetCustomStepsFromFiles(`before`),
const steps: ContainerHook[] = [
...ContainerHookService.ParseContainerHooks(CloudRunner.buildParameters.preBuildContainerHooks),
...ContainerHookService.GetContainerHooksFromFiles(`before`),
];
if (steps.length > 0) {
if (!CloudRunner.buildParameters.isCliMode) core.startGroup('pre build steps');
output += await CustomWorkflow.runCustomJob(
output += await CustomWorkflow.runContainerJob(
steps,
cloudRunnerStepState.environment,
cloudRunnerStepState.secrets,

4
src/model/cloud-runner/services/custom-step.ts → src/model/cloud-runner/services/hooks/container-hook.ts
View File

@ -1,6 +1,6 @@
import CloudRunnerSecret from './cloud-runner-secret';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
export class CustomStep {
export class ContainerHook {
public commands!: string;
public secrets: CloudRunnerSecret[] = new Array<CloudRunnerSecret>();
public name!: string;

47
src/model/cloud-runner/services/lfs-hashing.ts
View File

@ -1,47 +0,0 @@
import path from 'node:path';
import { CloudRunnerFolders } from './cloud-runner-folders';
import { CloudRunnerSystem } from './cloud-runner-system';
import fs from 'node:fs';
import { assert } from 'node:console';
import { Cli } from '../../cli/cli';
import { CliFunction } from '../../cli/cli-functions-repository';
export class LfsHashing {
public static async createLFSHashFiles() {
try {
await CloudRunnerSystem.Run(`git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-guid`);
await CloudRunnerSystem.Run(`md5sum .lfs-assets-guid > .lfs-assets-guid-sum`);
assert(fs.existsSync(`.lfs-assets-guid-sum`));
assert(fs.existsSync(`.lfs-assets-guid`));
const lfsHashes = {
lfsGuid: fs
.readFileSync(`${path.join(CloudRunnerFolders.repoPathAbsolute, `.lfs-assets-guid`)}`, 'utf8')
.replace(/\n/g, ``),
lfsGuidSum: fs
.readFileSync(`${path.join(CloudRunnerFolders.repoPathAbsolute, `.lfs-assets-guid-sum`)}`, 'utf8')
.replace(' .lfs-assets-guid', '')
.replace(/\n/g, ``),
};
return lfsHashes;
} catch (error) {
throw error;
}
}
public static async hashAllFiles(folder: string) {
const startPath = process.cwd();
process.chdir(folder);
const result = await (await CloudRunnerSystem.Run(`find -type f -exec md5sum "{}" + | sort | md5sum`))
.replace(/\n/g, '')
.split(` `)[0];
process.chdir(startPath);
return result;
}
@CliFunction(`hash`, `hash all folder contents`)
static async hash() {
const folder = Cli.options!['cachePushFrom'];
LfsHashing.hashAllFiles(folder);
}
}

43
src/model/cloud-runner/services/utility/lfs-hashing.ts Normal file
View File

@ -0,0 +1,43 @@
import path from 'node:path';
import { CloudRunnerFolders } from '../../options/cloud-runner-folders';
import { CloudRunnerSystem } from '../core/cloud-runner-system';
import fs from 'node:fs';
import { Cli } from '../../../cli/cli';
import { CliFunction } from '../../../cli/cli-functions-repository';
export class LfsHashing {
public static async createLFSHashFiles() {
await CloudRunnerSystem.Run(`git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-guid`);
await CloudRunnerSystem.Run(`md5sum .lfs-assets-guid > .lfs-assets-guid-sum`);
const lfsHashes = {
lfsGuid: fs
.readFileSync(`${path.join(CloudRunnerFolders.repoPathAbsolute, `.lfs-assets-guid`)}`, 'utf8')
.replace(/\n/g, ``),
lfsGuidSum: fs
.readFileSync(`${path.join(CloudRunnerFolders.repoPathAbsolute, `.lfs-assets-guid-sum`)}`, 'utf8')
.replace(' .lfs-assets-guid', '')
.replace(/\n/g, ``),
};
return lfsHashes;
}
public static async hashAllFiles(folder: string) {
const startPath = process.cwd();
process.chdir(folder);
const result = await (await CloudRunnerSystem.Run(`find -type f -exec md5sum "{}" + | sort | md5sum`))
.replace(/\n/g, '')
.split(` `)[0];
process.chdir(startPath);
return result;
}
@CliFunction(`hash`, `hash all folder contents`)
static async hash() {
if (!Cli.options) {
return;
}
const folder = Cli.options['cachePushFrom'];
LfsHashing.hashAllFiles(folder);
}
}

4
src/model/cloud-runner/tests/cloud-runner-async-workflow.test.ts
View File

@ -2,7 +2,7 @@ import { BuildParameters, ImageTag } from '../..';
import CloudRunner from '../cloud-runner';
import UnityVersioning from '../../unity-versioning';
import { Cli } from '../../cli/cli';
import CloudRunnerOptions from '../cloud-runner-options';
import CloudRunnerOptions from '../options/cloud-runner-options';
import setups from './cloud-runner-suite.test';
import { OptionValues } from 'commander';
@ -15,7 +15,7 @@ describe('Cloud Runner Async Workflows', () => {
setups();
it('Responds', () => {});
if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.cloudRunnerCluster !== `local-docker`) {
if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.providerStrategy !== `local-docker`) {
it('Async Workflows', async () => {
// Setup parameters
const buildParameter = await CreateParameters({

7
src/model/cloud-runner/tests/cloud-runner-remote-client-caching.test.ts → src/model/cloud-runner/tests/cloud-runner-caching.test.ts
View File

@ -4,14 +4,15 @@ import BuildParameters from '../../build-parameters';
import { Cli } from '../../cli/cli';
import UnityVersioning from '../../unity-versioning';
import CloudRunner from '../cloud-runner';
import { CloudRunnerSystem } from '../services/cloud-runner-system';
import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
import { Caching } from '../remote-client/caching';
import { v4 as uuidv4 } from 'uuid';
import GitHub from '../../github';
import CloudRunnerOptions from '../options/cloud-runner-options';
describe('Cloud Runner (Remote Client) Caching', () => {
it('responds', () => {});
if (process.platform === 'linux') {
it.skip('Simple caching works', async () => {
if (CloudRunnerOptions.providerStrategy === `local-docker`) {
it('Simple caching works', async () => {
Cli.options = {
versioning: 'None',
projectPath: 'test-project',

46
src/model/cloud-runner/tests/cloud-runner-environment-serializer.test.ts
View File

@ -1,46 +0,0 @@
import { BuildParameters } from '../..';
import { TaskParameterSerializer } from '../services/task-parameter-serializer';
import UnityVersioning from '../../unity-versioning';
import { Cli } from '../../cli/cli';
import GitHub from '../../github';
import setups from './cloud-runner-suite.test';
import { OptionValues } from 'commander';
async function CreateParameters(overrides: OptionValues | undefined) {
if (overrides) {
Cli.options = overrides;
}
const originalValue = GitHub.githubInputEnabled;
GitHub.githubInputEnabled = false;
const results = await BuildParameters.create();
GitHub.githubInputEnabled = originalValue;
delete Cli.options;
return results;
}
describe('Cloud Runner Environment Serializer', () => {
setups();
const testSecretName = 'testSecretName';
const testSecretValue = 'testSecretValue';
it('Cloud Runner Parameter Serialization', async () => {
// Setup parameters
const buildParameter = await CreateParameters({
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.read('test-project'),
customJob: `
- name: 'step 1'
image: 'alpine'
commands: 'printenv'
secrets:
- name: '${testSecretName}'
value: '${testSecretValue}'
`,
});
const result = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter);
expect(result.find((x) => Number.parseInt(x.name)) !== undefined).toBeFalsy();
const result2 = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter);
expect(result2.find((x) => Number.parseInt(x.name)) !== undefined).toBeFalsy();
});
});

57
src/model/cloud-runner/tests/cloud-runner-sync-environment.test.ts → src/model/cloud-runner/tests/cloud-runner-environment.test.ts
View File

@ -1,20 +1,26 @@
import { BuildParameters, ImageTag } from '../..';
import CloudRunner from '../cloud-runner';
import Input from '../../input';
import { CloudRunnerStatics } from '../cloud-runner-statics';
import { TaskParameterSerializer } from '../services/task-parameter-serializer';
import { BuildParameters, CloudRunner, ImageTag, Input } from '../..';
import { TaskParameterSerializer } from '../services/core/task-parameter-serializer';
import UnityVersioning from '../../unity-versioning';
import { Cli } from '../../cli/cli';
import CloudRunnerLogger from '../services/cloud-runner-logger';
import CloudRunnerOptions from '../cloud-runner-options';
import GitHub from '../../github';
import setups from './cloud-runner-suite.test';
import { OptionValues } from 'commander';
import { CloudRunnerStatics } from '../options/cloud-runner-statics';
import CloudRunnerOptions from '../options/cloud-runner-options';
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
async function CreateParameters(overrides: OptionValues | undefined) {
if (overrides) Cli.options = overrides;
async function CreateParameters(overrides: any) {
if (overrides) {
Cli.options = overrides;
}
const originalValue = GitHub.githubInputEnabled;
GitHub.githubInputEnabled = false;
const results = await BuildParameters.create();
GitHub.githubInputEnabled = originalValue;
delete Cli.options;
return BuildParameters.create();
return results;
}
describe('Cloud Runner Sync Environments', () => {
setups();
const testSecretName = 'testSecretName';
@ -62,7 +68,7 @@ describe('Cloud Runner Sync Environments', () => {
return x;
})
.filter((element) => {
return !['UNITY_LICENSE', 'CUSTOM_JOB'].includes(element.name);
return !['UNITY_LICENSE', 'UNITY_LICENSE', 'CUSTOM_JOB', 'CUSTOM_JOB'].includes(element.name);
});
const newLinePurgedFile = file
.replace(/\s+/g, '')
@ -76,3 +82,30 @@ describe('Cloud Runner Sync Environments', () => {
}, 1_000_000_000);
}
});
describe('Cloud Runner Environment Serializer', () => {
setups();
const testSecretName = 'testSecretName';
const testSecretValue = 'testSecretValue';
it('Cloud Runner Parameter Serialization', async () => {
// Setup parameters
const buildParameter = await CreateParameters({
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.read('test-project'),
customJob: `
- name: 'step 1'
image: 'alpine'
commands: 'printenv'
secrets:
- name: '${testSecretName}'
value: '${testSecretValue}'
`,
});
const result = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter);
expect(result.find((x) => Number.parseInt(x.name)) !== undefined).toBeFalsy();
const result2 = TaskParameterSerializer.createCloudRunnerEnvironmentVariables(buildParameter);
expect(result2.find((x) => Number.parseInt(x.name)) !== undefined).toBeFalsy();
});
});

114
src/model/cloud-runner/tests/cloud-runner-hooks.test.ts Normal file
View File

@ -0,0 +1,114 @@
import CloudRunner from '../cloud-runner';
import { BuildParameters, ImageTag } from '../..';
import UnityVersioning from '../../unity-versioning';
import { Cli } from '../../cli/cli';
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
import { v4 as uuidv4 } from 'uuid';
import CloudRunnerOptions from '../options/cloud-runner-options';
import setups from './cloud-runner-suite.test';
import { ContainerHookService } from '../services/hooks/container-hook-service';
import { CommandHookService } from '../services/hooks/command-hook-service';
async function CreateParameters(overrides: any) {
if (overrides) {
Cli.options = overrides;
}
return await BuildParameters.create();
}
describe('Cloud Runner Custom Hooks And Steps', () => {
it('Responds', () => {});
setups();
it('Check parsing and reading of steps', async () => {
const yamlString = `hook: before
commands: echo "test"`;
const yamlString2 = `- hook: before
commands: echo "test"`;
const overrides = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
};
CloudRunner.setup(await CreateParameters(overrides));
const stringObject = ContainerHookService.ParseContainerHooks(yamlString);
const stringObject2 = ContainerHookService.ParseContainerHooks(yamlString2);
CloudRunnerLogger.log(yamlString);
CloudRunnerLogger.log(JSON.stringify(stringObject, undefined, 4));
expect(stringObject.length).toBe(1);
expect(stringObject[0].hook).toBe(`before`);
expect(stringObject2.length).toBe(1);
expect(stringObject2[0].hook).toBe(`before`);
const getCustomStepsFromFiles = ContainerHookService.GetContainerHooksFromFiles(`before`);
CloudRunnerLogger.log(JSON.stringify(getCustomStepsFromFiles, undefined, 4));
});
if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.providerStrategy !== `k8s`) {
it('Should be 1 before and 1 after hook', async () => {
const overrides = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
containerHookFiles: `my-test-step-pre-build,my-test-step-post-build`,
commandHookFiles: `my-test-hook-pre-build,my-test-hook-post-build`,
};
const buildParameter2 = await CreateParameters(overrides);
await CloudRunner.setup(buildParameter2);
const beforeHooks = CommandHookService.GetCustomHooksFromFiles(`before`);
const afterHooks = CommandHookService.GetCustomHooksFromFiles(`after`);
expect(beforeHooks).toHaveLength(1);
expect(afterHooks).toHaveLength(1);
});
it('Should be 1 before and 1 after step', async () => {
const overrides = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
containerHookFiles: `my-test-step-pre-build,my-test-step-post-build`,
commandHookFiles: `my-test-hook-pre-build,my-test-hook-post-build`,
};
const buildParameter2 = await CreateParameters(overrides);
await CloudRunner.setup(buildParameter2);
const beforeSteps = ContainerHookService.GetContainerHooksFromFiles(`before`);
const afterSteps = ContainerHookService.GetContainerHooksFromFiles(`after`);
expect(beforeSteps).toHaveLength(1);
expect(afterSteps).toHaveLength(1);
});
it('Run build once - check for pre and post custom hooks run contents', async () => {
const overrides = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
containerHookFiles: `my-test-step-pre-build,my-test-step-post-build`,
commandHookFiles: `my-test-hook-pre-build,my-test-hook-post-build`,
};
const buildParameter2 = await CreateParameters(overrides);
const baseImage2 = new ImageTag(buildParameter2);
const results2 = await CloudRunner.run(buildParameter2, baseImage2.toString());
CloudRunnerLogger.log(`run 2 succeeded`);
const buildContainsBuildSucceeded = results2.includes('Build succeeded');
const buildContainsPreBuildHookRunMessage = results2.includes('before-build hook test!');
const buildContainsPostBuildHookRunMessage = results2.includes('after-build hook test!');
const buildContainsPreBuildStepMessage = results2.includes('before-build step test!');
const buildContainsPostBuildStepMessage = results2.includes('after-build step test!');
expect(buildContainsBuildSucceeded).toBeTruthy();
expect(buildContainsPreBuildHookRunMessage).toBeTruthy();
expect(buildContainsPostBuildHookRunMessage).toBeTruthy();
expect(buildContainsPreBuildStepMessage).toBeTruthy();
expect(buildContainsPostBuildStepMessage).toBeTruthy();
}, 1_000_000_000);
}
});

53
src/model/cloud-runner/tests/cloud-runner-local-persistence.test.ts Normal file
View File

@ -0,0 +1,53 @@
import { ImageTag } from '../..';
import CloudRunner from '../cloud-runner';
import UnityVersioning from '../../unity-versioning';
import CloudRunnerOptions from '../options/cloud-runner-options';
import setups from './cloud-runner-suite.test';
import fs from 'node:fs';
import { CreateParameters } from './create-test-parameter';
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
describe('Cloud Runner Local Docker Workflows', () => {
setups();
it('Responds', () => {});
if (CloudRunnerOptions.providerStrategy === `local-docker`) {
it('inspect stateful folder of workflows', async () => {
const testValue = `the state in a job exits in the expected local-docker folder`;
// Setup parameters
const buildParameter = await CreateParameters({
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.read('test-project'),
customJob: `
- name: 'step 1'
image: 'ubuntu'
commands: 'echo "${testValue}" >> /data/test-out-state.txt'
`,
});
const buildParameter2 = await CreateParameters({
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.read('test-project'),
customJob: `
- name: 'step 1'
image: 'ubuntu'
commands: 'cat /data/test-out-state.txt >> /data/test-out-state-2.txt'
`,
});
const baseImage = new ImageTag(buildParameter);
// Run the job
await CloudRunner.run(buildParameter, baseImage.toString());
await CloudRunner.run(buildParameter2, baseImage.toString());
const outputFile = fs.readFileSync(`./cloud-runner-cache/test-out-state.txt`, `utf-8`);
expect(outputFile).toMatch(testValue);
const outputFile2 = fs.readFileSync(`./cloud-runner-cache/test-out-state-2.txt`, `utf-8`);
expect(outputFile2).toMatch(testValue);
CloudRunnerLogger.log(outputFile);
}, 1_000_000_000);
}
});

115
src/model/cloud-runner/tests/cloud-runner-locking-core.test.ts Normal file
View File

@ -0,0 +1,115 @@
import SharedWorkspaceLocking from '../services/core/shared-workspace-locking';
import { Cli } from '../../cli/cli';
import setups from './cloud-runner-suite.test';
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
import { v4 as uuidv4 } from 'uuid';
import CloudRunnerOptions from '../options/cloud-runner-options';
import UnityVersioning from '../../unity-versioning';
import BuildParameters from '../../build-parameters';
import CloudRunner from '../cloud-runner';
async function CreateParameters(overrides: any) {
if (overrides) {
Cli.options = overrides;
}
return await BuildParameters.create();
}
describe('Cloud Runner Locking Core', () => {
setups();
it('Responds', () => {});
if (CloudRunnerOptions.cloudRunnerDebug) {
it(`Create Workspace`, async () => {
const overrides: any = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
maxRetainedWorkspaces: 3,
};
const buildParameters = await CreateParameters(overrides);
CloudRunner.buildParameters = buildParameters;
const newWorkspaceName = `test-workspace-${uuidv4()}`;
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
}, 150000);
it(`Create Workspace And Lock Workspace`, async () => {
const overrides: any = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
maxRetainedWorkspaces: 3,
};
const runId = uuidv4();
const buildParameters = await CreateParameters(overrides);
CloudRunner.buildParameters = buildParameters;
const newWorkspaceName = `test-workspace-${uuidv4()}`;
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
}, 150000);
it(`0 free workspaces after locking`, async () => {
const overrides: any = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
maxRetainedWorkspaces: 3,
};
const buildParameters = await CreateParameters(overrides);
const newWorkspaceName = `test-workspace-${uuidv4()}`;
const runId = uuidv4();
CloudRunner.buildParameters = buildParameters;
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.HasWorkspaceLock(newWorkspaceName, runId, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.GetAllWorkspaces(buildParameters)).toHaveLength(1);
expect(await SharedWorkspaceLocking.GetAllLocksForWorkspace(newWorkspaceName, buildParameters)).toHaveLength(1);
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeTruthy();
const files = await SharedWorkspaceLocking.ReadLines(
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParameters.cacheKey}/`,
);
const lockFilesExist =
files.filter((x) => {
return x.includes(newWorkspaceName) && x.endsWith(`_lock`);
}).length > 0;
expect(files).toHaveLength(2);
expect(
files.filter((x) => {
return x.includes(newWorkspaceName) && x.endsWith(`_lock`);
}),
).toHaveLength(1);
expect(lockFilesExist).toBeTruthy();
const result: string[] = [];
const workspaces = await SharedWorkspaceLocking.GetAllWorkspaces(buildParameters);
for (const element of workspaces) {
expect((await SharedWorkspaceLocking.GetAllWorkspaces(buildParameters)).join()).toContain(element);
expect(await SharedWorkspaceLocking.GetAllWorkspaces(buildParameters)).toHaveLength(1);
expect(await SharedWorkspaceLocking.DoesWorkspaceExist(element, buildParameters)).toBeTruthy();
await new Promise((promise) => setTimeout(promise, 1500));
const isLocked = await SharedWorkspaceLocking.IsWorkspaceLocked(element, buildParameters);
const isBelowMax = await SharedWorkspaceLocking.IsWorkspaceBelowMax(element, buildParameters);
CloudRunnerLogger.log(`workspace ${element} locked:${isLocked} below max:${isBelowMax}`);
const lock = files.find((x) => {
return x.endsWith(`_lock`);
});
expect(lock).toContain(element);
expect(isLocked).toBeTruthy();
expect(isBelowMax).toBeTruthy();
if (!isLocked && isBelowMax) {
result.push(element);
}
}
expect(result).toHaveLength(0);
expect(await SharedWorkspaceLocking.GetFreeWorkspaces(buildParameters)).toHaveLength(0);
}, 300000);
}
});

156
src/model/cloud-runner/tests/cloud-runner-locking-get-locked.test.ts Normal file
View File

@ -0,0 +1,156 @@
import SharedWorkspaceLocking from '../services/core/shared-workspace-locking';
import { Cli } from '../../cli/cli';
import setups from './cloud-runner-suite.test';
import { v4 as uuidv4 } from 'uuid';
import CloudRunnerOptions from '../options/cloud-runner-options';
import UnityVersioning from '../../unity-versioning';
import BuildParameters from '../../build-parameters';
import CloudRunner from '../cloud-runner';
async function CreateParameters(overrides: any) {
if (overrides) {
Cli.options = overrides;
}
return await BuildParameters.create();
}
describe('Cloud Runner Locking Get Locked Workspace', () => {
setups();
it('Responds', () => {});
if (CloudRunnerOptions.cloudRunnerDebug) {
it(`Get locked workspace From No Workspace`, async () => {
const overrides: any = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
maxRetainedWorkspaces: 3,
};
const buildParameters = await CreateParameters(overrides);
const newWorkspaceName = `test-workspace-${uuidv4()}`;
const runId = uuidv4();
CloudRunner.buildParameters = buildParameters;
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
}, 150000);
it(`Get locked workspace from unlocked`, async () => {
const overrides: any = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
maxRetainedWorkspaces: 3,
};
const buildParameters = await CreateParameters(overrides);
const newWorkspaceName = `test-workspace-${uuidv4()}`;
const runId = uuidv4();
CloudRunner.buildParameters = buildParameters;
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
expect(CloudRunner.lockedWorkspace).toMatch(newWorkspaceName);
}, 300000);
it(`Get locked workspace from locked`, async () => {
const overrides: any = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
maxRetainedWorkspaces: 3,
};
const buildParameters = await CreateParameters(overrides);
const newWorkspaceName = `test-workspace-${uuidv4()}`;
const runId = uuidv4();
const runId2 = uuidv4();
CloudRunner.buildParameters = buildParameters;
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.HasWorkspaceLock(newWorkspaceName, runId, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.IsWorkspaceBelowMax(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId2, buildParameters)).toBeTruthy();
expect(CloudRunner.lockedWorkspace).not.toMatch(newWorkspaceName);
}, 300000);
it(`Get locked workspace after double lock and one unlock`, async () => {
const overrides: any = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
maxRetainedWorkspaces: 3,
};
const buildParameters = await CreateParameters(overrides);
const newWorkspaceName = `test-workspace-${uuidv4()}`;
const runId = uuidv4();
const runId2 = uuidv4();
CloudRunner.buildParameters = buildParameters;
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeFalsy();
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.HasWorkspaceLock(newWorkspaceName, runId, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId2, buildParameters)).toBeTruthy();
expect(CloudRunner.lockedWorkspace).not.toContain(newWorkspaceName);
}, 300000);
it(`Get locked workspace after double lock and unlock`, async () => {
const overrides: any = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
maxRetainedWorkspaces: 3,
};
const buildParameters = await CreateParameters(overrides);
const newWorkspaceName = `test-workspace-${uuidv4()}`;
const runId = uuidv4();
const runId2 = uuidv4();
CloudRunner.buildParameters = buildParameters;
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeFalsy();
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.HasWorkspaceLock(newWorkspaceName, runId, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)).toBeFalsy();
expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId2, buildParameters)).toBeTruthy();
expect(CloudRunner.lockedWorkspace).toContain(newWorkspaceName);
}, 300000);
it(`Get locked workspace from unlocked was locked`, async () => {
const overrides: any = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
maxRetainedWorkspaces: 3,
};
const buildParameters = await CreateParameters(overrides);
const newWorkspaceName = `test-workspace-${uuidv4()}`;
const runId = uuidv4();
CloudRunner.buildParameters = buildParameters;
expect(await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
expect(CloudRunner.lockedWorkspace).toMatch(newWorkspaceName);
}, 300000);
}
});

79
src/model/cloud-runner/tests/cloud-runner-run-once-custom-hooks.test.ts
View File

@ -1,79 +0,0 @@
import CloudRunner from '../cloud-runner';
import { BuildParameters, ImageTag } from '../..';
import UnityVersioning from '../../unity-versioning';
import { Cli } from '../../cli/cli';
import CloudRunnerLogger from '../services/cloud-runner-logger';
import { v4 as uuidv4 } from 'uuid';
import CloudRunnerOptions from '../cloud-runner-options';
import setups from './cloud-runner-suite.test';
import { CloudRunnerCustomSteps } from '../services/cloud-runner-custom-steps';
import { OptionValues } from 'commander';
async function CreateParameters(overrides: OptionValues | undefined) {
if (overrides) {
Cli.options = overrides;
}
return await BuildParameters.create();
}
describe('Cloud Runner Custom Hooks And Steps', () => {
it('Responds', () => {});
setups();
it('Check parsing and reading of steps', async () => {
const yamlString = `hook: before
commands: echo "test"`;
const yamlString2 = `- hook: before
commands: echo "test"`;
const overrides = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
};
CloudRunner.setup(await CreateParameters(overrides));
const stringObject = CloudRunnerCustomSteps.ParseSteps(yamlString);
const stringObject2 = CloudRunnerCustomSteps.ParseSteps(yamlString2);
CloudRunnerLogger.log(yamlString);
CloudRunnerLogger.log(JSON.stringify(stringObject, undefined, 4));
expect(stringObject.length).toBe(1);
expect(stringObject[0].hook).toBe(`before`);
expect(stringObject2.length).toBe(1);
expect(stringObject2[0].hook).toBe(`before`);
const getCustomStepsFromFiles = CloudRunnerCustomSteps.GetCustomStepsFromFiles(`before`);
CloudRunnerLogger.log(JSON.stringify(getCustomStepsFromFiles, undefined, 4));
});
if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.cloudRunnerCluster !== `k8s`) {
it('Run build once - check for pre and post custom hooks run contents', async () => {
const overrides = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
customStepFiles: `my-test-step-pre-build,my-test-step-post-build`,
};
const buildParameter2 = await CreateParameters(overrides);
const baseImage2 = new ImageTag(buildParameter2);
const results2 = await CloudRunner.run(buildParameter2, baseImage2.toString());
CloudRunnerLogger.log(`run 2 succeeded`);
const build2ContainsBuildSucceeded = results2.includes('Build succeeded');
const build2ContainsPreBuildHookRunMessage = results2.includes('before-build hook test!');
const build2ContainsPostBuildHookRunMessage = results2.includes('after-build hook test!');
const build2ContainsPreBuildStepMessage = results2.includes('before-build step test!');
const build2ContainsPostBuildStepMessage = results2.includes('after-build step test!');
expect(build2ContainsBuildSucceeded).toBeTruthy();
expect(build2ContainsPreBuildHookRunMessage).toBeTruthy();
expect(build2ContainsPostBuildHookRunMessage).toBeTruthy();
expect(build2ContainsPreBuildStepMessage).toBeTruthy();
expect(build2ContainsPostBuildStepMessage).toBeTruthy();
}, 1_000_000_000);
}
});

12
src/model/cloud-runner/tests/cloud-runner-s3-prebuilt-steps.test.ts → src/model/cloud-runner/tests/cloud-runner-s3-steps.test.ts
View File

@ -2,11 +2,11 @@ import CloudRunner from '../cloud-runner';
import { BuildParameters, ImageTag } from '../..';
import UnityVersioning from '../../unity-versioning';
import { Cli } from '../../cli/cli';
import CloudRunnerLogger from '../services/cloud-runner-logger';
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
import { v4 as uuidv4 } from 'uuid';
import CloudRunnerOptions from '../cloud-runner-options';
import CloudRunnerOptions from '../options/cloud-runner-options';
import setups from './cloud-runner-suite.test';
import { CloudRunnerSystem } from '../services/cloud-runner-system';
import { CloudRunnerSystem } from '../services/core/cloud-runner-system';
import { OptionValues } from 'commander';
async function CreateParameters(overrides: OptionValues | undefined) {
@ -20,7 +20,7 @@ async function CreateParameters(overrides: OptionValues | undefined) {
describe('Cloud Runner pre-built S3 steps', () => {
it('Responds', () => {});
setups();
if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.cloudRunnerCluster !== `local-docker`) {
if (CloudRunnerOptions.cloudRunnerDebug && CloudRunnerOptions.providerStrategy !== `local-docker`) {
it('Run build and prebuilt s3 cache pull, cache push and upload build', async () => {
const overrides = {
versioning: 'None',
@ -28,7 +28,7 @@ describe('Cloud Runner pre-built S3 steps', () => {
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
customStepFiles: `aws-s3-pull-cache,aws-s3-upload-cache,aws-s3-upload-build`,
containerHookFiles: `aws-s3-pull-cache,aws-s3-upload-cache,aws-s3-upload-build`,
};
const buildParameter2 = await CreateParameters(overrides);
const baseImage2 = new ImageTag(buildParameter2);
@ -39,7 +39,7 @@ describe('Cloud Runner pre-built S3 steps', () => {
expect(build2ContainsBuildSucceeded).toBeTruthy();
const results = await CloudRunnerSystem.RunAndReadLines(
`aws s3 ls s3://${CloudRunner.buildParameters.awsBaseStackName}/cloud-runner-cache/${buildParameter2.cacheKey}/`,
`aws s3 ls s3://${CloudRunner.buildParameters.awsStackName}/cloud-runner-cache/`,
);
CloudRunnerLogger.log(results.join(`,`));
}, 1_000_000_000);

8
src/model/cloud-runner/tests/create-test-parameter.ts Normal file
View File

@ -0,0 +1,8 @@
import BuildParameters from '../../build-parameters';
import { Cli } from '../../cli/cli';
export async function CreateParameters(overrides: any) {
if (overrides) Cli.options = overrides;
return BuildParameters.create();
}

35
src/model/cloud-runner/tests/cloud-runner-run-twice-caching.test.ts → src/model/cloud-runner/tests/e2e/cloud-runner-end2end-caching.test.ts
View File

@ -1,15 +1,15 @@
import CloudRunner from '../cloud-runner';
import { BuildParameters, ImageTag } from '../..';
import UnityVersioning from '../../unity-versioning';
import { Cli } from '../../cli/cli';
import CloudRunnerLogger from '../services/cloud-runner-logger';
import CloudRunner from '../../cloud-runner';
import { BuildParameters, ImageTag } from '../../..';
import UnityVersioning from '../../../unity-versioning';
import { Cli } from '../../../cli/cli';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { v4 as uuidv4 } from 'uuid';
import CloudRunnerOptions from '../cloud-runner-options';
import setups from './cloud-runner-suite.test';
import fs from 'node:fs';
import { OptionValues } from 'commander';
import CloudRunnerOptions from '../../options/cloud-runner-options';
import setups from '../cloud-runner-suite.test';
import * as fs from 'node:fs';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
async function CreateParameters(overrides: OptionValues | undefined) {
async function CreateParameters(overrides: any) {
if (overrides) {
Cli.options = overrides;
}
@ -28,10 +28,10 @@ describe('Cloud Runner Caching', () => {
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
customStepFiles: `debug-cache`,
containerHookFiles: `debug-cache`,
};
if (CloudRunnerOptions.cloudRunnerCluster === `k8s`) {
overrides.customStepFiles += `,aws-s3-pull-cache,aws-s3-upload-cache`;
if (CloudRunnerOptions.providerStrategy === `k8s`) {
overrides.containerHookFiles += `,aws-s3-pull-cache,aws-s3-upload-cache`;
}
const buildParameter = await CreateParameters(overrides);
expect(buildParameter.projectPath).toEqual(overrides.projectPath);
@ -48,7 +48,14 @@ describe('Cloud Runner Caching', () => {
CloudRunnerLogger.log(`run 1 succeeded`);
if (CloudRunnerOptions.cloudRunnerCluster === `local-docker`) {
if (CloudRunnerOptions.providerStrategy === `local-docker`) {
await CloudRunnerSystem.Run(`tree ./cloud-runner-cache/cache`);
await CloudRunnerSystem.Run(
`cp ./cloud-runner-cache/cache/${buildParameter.cacheKey}/Library/lib-${buildParameter.buildGuid}.tar ./`,
);
await CloudRunnerSystem.Run(`mkdir results`);
await CloudRunnerSystem.Run(`tar -xf lib-${buildParameter.buildGuid}.tar -C ./results`);
await CloudRunnerSystem.Run(`tree -d ./results`);
const cacheFolderExists = fs.existsSync(`cloud-runner-cache/cache/${overrides.cacheKey}`);
expect(cacheFolderExists).toBeTruthy();
}

92
src/model/cloud-runner/tests/e2e/cloud-runner-end2end-locking.test.ts Normal file
View File

@ -0,0 +1,92 @@
import CloudRunner from '../../cloud-runner';
import { BuildParameters } from '../../..';
import UnityVersioning from '../../../unity-versioning';
import { Cli } from '../../../cli/cli';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { v4 as uuidv4 } from 'uuid';
import CloudRunnerOptions from '../../options/cloud-runner-options';
import setups from '../cloud-runner-suite.test';
import SharedWorkspaceLocking from '../../services/core/shared-workspace-locking';
async function CreateParameters(overrides: any) {
if (overrides) {
Cli.options = overrides;
}
return await BuildParameters.create();
}
describe('Cloud Runner Locking', () => {
setups();
it('Responds', () => {});
if (CloudRunnerOptions.cloudRunnerDebug) {
it(`Simple Locking End2End Flow`, async () => {
const overrides: any = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
maxRetainedWorkspaces: 3,
};
const buildParameters = await CreateParameters(overrides);
const newWorkspaceName = `test-workspace-${uuidv4()}`;
const runId = uuidv4();
CloudRunner.buildParameters = buildParameters;
await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters);
expect(await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
const isExpectedUnlockedBeforeLocking =
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === false;
expect(isExpectedUnlockedBeforeLocking).toBeTruthy();
const result = await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters);
expect(result).toBeTruthy();
const lines = await SharedWorkspaceLocking.ReadLines(`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}`);
expect(lines.map((x) => x.replace(`/`, ``)).includes(buildParameters.cacheKey));
expect(await SharedWorkspaceLocking.DoesCacheKeyTopLevelExist(buildParameters)).toBeTruthy();
expect(await SharedWorkspaceLocking.DoesWorkspaceExist(newWorkspaceName, buildParameters)).toBeTruthy();
const allLocks = await SharedWorkspaceLocking.GetAllLocksForWorkspace(newWorkspaceName, buildParameters);
expect(
(
await SharedWorkspaceLocking.ReadLines(
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParameters.cacheKey}/`,
)
).filter((x) => x.endsWith(`${newWorkspaceName}_workspace_lock`)),
).toHaveLength(1);
expect(
(
await SharedWorkspaceLocking.ReadLines(
`aws s3 ls ${SharedWorkspaceLocking.workspaceRoot}${buildParameters.cacheKey}/`,
)
).filter((x) => x.endsWith(`${newWorkspaceName}_workspace`)),
).toHaveLength(1);
expect(allLocks.filter((x) => x.endsWith(`${newWorkspaceName}_workspace_lock`)).length).toBeGreaterThan(0);
const isExpectedLockedAfterLocking =
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === true;
expect(isExpectedLockedAfterLocking).toBeTruthy();
const locksBeforeRelease = await SharedWorkspaceLocking.GetAllLocksForWorkspace(
newWorkspaceName,
buildParameters,
);
CloudRunnerLogger.log(JSON.stringify(locksBeforeRelease, undefined, 4));
expect(locksBeforeRelease.length).toBe(1);
await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters);
const locks = await SharedWorkspaceLocking.GetAllLocksForWorkspace(newWorkspaceName, buildParameters);
expect(locks.length).toBe(0);
const isExpectedNotLockedAfterReleasing =
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === false;
expect(isExpectedNotLockedAfterReleasing).toBeTruthy();
const lockingResult2 = await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters);
expect(lockingResult2).toBeTruthy();
expect((await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === true).toBeTruthy();
await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters);
expect(
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === false,
).toBeTruthy();
await SharedWorkspaceLocking.CleanupWorkspace(newWorkspaceName, buildParameters);
CloudRunnerLogger.log(`Starting get or create`);
expect(await SharedWorkspaceLocking.GetLockedWorkspace(newWorkspaceName, runId, buildParameters)).toBeTruthy();
}, 350000);
}
});

37
src/model/cloud-runner/tests/cloud-runner-run-twice-retaining.test.ts → src/model/cloud-runner/tests/e2e/cloud-runner-end2end-retaining.test.ts
View File

@ -1,24 +1,16 @@
import CloudRunner from '../cloud-runner';
import { BuildParameters, ImageTag } from '../..';
import UnityVersioning from '../../unity-versioning';
import { Cli } from '../../cli/cli';
import CloudRunnerLogger from '../services/cloud-runner-logger';
import CloudRunner from '../../cloud-runner';
import { ImageTag } from '../../..';
import UnityVersioning from '../../../unity-versioning';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { v4 as uuidv4 } from 'uuid';
import CloudRunnerOptions from '../cloud-runner-options';
import setups from './cloud-runner-suite.test';
import fs from 'node:fs';
import CloudRunnerOptions from '../../options/cloud-runner-options';
import setups from './../cloud-runner-suite.test';
import * as fs from 'node:fs';
import path from 'node:path';
import { CloudRunnerFolders } from '../services/cloud-runner-folders';
import SharedWorkspaceLocking from '../services/shared-workspace-locking';
import { OptionValues } from 'commander';
async function CreateParameters(overrides: OptionValues | undefined) {
if (overrides) {
Cli.options = overrides;
}
return await BuildParameters.create();
}
import { CloudRunnerFolders } from '../../options/cloud-runner-folders';
import SharedWorkspaceLocking from '../../services/core/shared-workspace-locking';
import { CreateParameters } from '../create-test-parameter';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
describe('Cloud Runner Retain Workspace', () => {
it('Responds', () => {});
@ -31,7 +23,7 @@ describe('Cloud Runner Retain Workspace', () => {
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
retainWorkspaces: true,
maxRetainedWorkspaces: 1,
};
const buildParameter = await CreateParameters(overrides);
expect(buildParameter.projectPath).toEqual(overrides.projectPath);
@ -46,12 +38,15 @@ describe('Cloud Runner Retain Workspace', () => {
expect(results).toContain(buildSucceededString);
expect(results).not.toContain(cachePushFail);
if (CloudRunnerOptions.cloudRunnerCluster === `local-docker`) {
if (CloudRunnerOptions.providerStrategy === `local-docker`) {
const cacheFolderExists = fs.existsSync(`cloud-runner-cache/cache/${overrides.cacheKey}`);
expect(cacheFolderExists).toBeTruthy();
await CloudRunnerSystem.Run(`tree -d ./cloud-runner-cache`);
}
CloudRunnerLogger.log(`run 1 succeeded`);
// await CloudRunnerSystem.Run(`tree -d ./cloud-runner-cache/${}`);
const buildParameter2 = await CreateParameters(overrides);
buildParameter2.cacheKey = buildParameter.cacheKey;

102
src/model/cloud-runner/tests/shared-workspace-locking.test.ts
View File

@ -1,102 +0,0 @@
import SharedWorkspaceLocking from '../services/shared-workspace-locking';
import { Cli } from '../../cli/cli';
import setups from './cloud-runner-suite.test';
import CloudRunnerLogger from '../services/cloud-runner-logger';
import { v4 as uuidv4 } from 'uuid';
import CloudRunnerOptions from '../cloud-runner-options';
import UnityVersioning from '../../unity-versioning';
import BuildParameters from '../../build-parameters';
import CloudRunner from '../cloud-runner';
import { OptionValues } from 'commander';
async function CreateParameters(overrides: OptionValues | undefined) {
if (overrides) {
Cli.options = overrides;
}
return await BuildParameters.create();
}
describe('Cloud Runner Locking', () => {
setups();
it('Responds', () => {});
if (CloudRunnerOptions.cloudRunnerDebug) {
it(`Simple Locking Flow`, async () => {
Cli.options!.retainWorkspaces = true;
const overrides: any = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
};
const buildParameters = await CreateParameters(overrides);
const newWorkspaceName = `test-workspace-${uuidv4()}`;
const runId = uuidv4();
CloudRunner.buildParameters = buildParameters;
await SharedWorkspaceLocking.CreateWorkspace(newWorkspaceName, buildParameters);
const isExpectedUnlockedBeforeLocking =
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === false;
expect(isExpectedUnlockedBeforeLocking).toBeTruthy();
await SharedWorkspaceLocking.LockWorkspace(newWorkspaceName, runId, buildParameters);
const isExpectedLockedAfterLocking =
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === true;
expect(isExpectedLockedAfterLocking).toBeTruthy();
const locksBeforeRelease = await SharedWorkspaceLocking.GetAllLocks(newWorkspaceName, buildParameters);
CloudRunnerLogger.log(JSON.stringify(locksBeforeRelease, undefined, 4));
expect(locksBeforeRelease.length).toBe(1);
await SharedWorkspaceLocking.ReleaseWorkspace(newWorkspaceName, runId, buildParameters);
const locks = await SharedWorkspaceLocking.GetAllLocks(newWorkspaceName, buildParameters);
expect(locks.length).toBe(0);
const isExpectedLockedAfterReleasing =
(await SharedWorkspaceLocking.IsWorkspaceLocked(newWorkspaceName, buildParameters)) === false;
expect(isExpectedLockedAfterReleasing).toBeTruthy();
}, 150000);
it.skip('All Locking Actions', async () => {
Cli.options!.retainWorkspaces = true;
const overrides: OptionValues = {
versioning: 'None',
projectPath: 'test-project',
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
targetPlatform: 'StandaloneLinux64',
cacheKey: `test-case-${uuidv4()}`,
};
const buildParameters = await CreateParameters(overrides);
CloudRunnerLogger.log(
`GetAllWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetAllWorkspaces(buildParameters))}`,
);
CloudRunnerLogger.log(
`GetFreeWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetFreeWorkspaces(buildParameters))}`,
);
CloudRunnerLogger.log(
`IsWorkspaceLocked ${JSON.stringify(
await SharedWorkspaceLocking.IsWorkspaceLocked(`test-workspace-${uuidv4()}`, buildParameters),
)}`,
);
CloudRunnerLogger.log(
`GetFreeWorkspaces ${JSON.stringify(await SharedWorkspaceLocking.GetFreeWorkspaces(buildParameters))}`,
);
CloudRunnerLogger.log(
`LockWorkspace ${JSON.stringify(
await SharedWorkspaceLocking.LockWorkspace(`test-workspace-${uuidv4()}`, uuidv4(), buildParameters),
)}`,
);
CloudRunnerLogger.log(
`CreateLockableWorkspace ${JSON.stringify(
await SharedWorkspaceLocking.CreateWorkspace(`test-workspace-${uuidv4()}`, buildParameters),
)}`,
);
CloudRunnerLogger.log(
`GetLockedWorkspace ${JSON.stringify(
await SharedWorkspaceLocking.GetOrCreateLockedWorkspace(
`test-workspace-${uuidv4()}`,
uuidv4(),
buildParameters,
),
)}`,
);
}, 3000000);
}
});

13
src/model/cloud-runner/workflows/async-workflow.ts
View File

@ -1,7 +1,7 @@
import CloudRunnerSecret from '../services/cloud-runner-secret';
import CloudRunnerEnvironmentVariable from '../services/cloud-runner-environment-variable';
import CloudRunnerLogger from '../services/cloud-runner-logger';
import { CloudRunnerFolders } from '../services/cloud-runner-folders';
import CloudRunnerSecret from '../options/cloud-runner-secret';
import CloudRunnerEnvironmentVariable from '../options/cloud-runner-environment-variable';
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
import { CloudRunnerFolders } from '../options/cloud-runner-folders';
import CloudRunner from '../cloud-runner';
export class AsyncWorkflow {
@ -11,6 +11,9 @@ export class AsyncWorkflow {
): Promise<string> {
try {
CloudRunnerLogger.log(`Cloud Runner is running async mode`);
const asyncEnvironmentVariable = new CloudRunnerEnvironmentVariable();
asyncEnvironmentVariable.name = `ASYNC_WORKFLOW`;
asyncEnvironmentVariable.value = `true`;
let output = '';
@ -34,7 +37,7 @@ aws --version
node /builder/dist/index.js -m async-workflow`,
`/${CloudRunnerFolders.buildVolumeFolder}`,
`/${CloudRunnerFolders.buildVolumeFolder}/`,
environmentVariables,
[...environmentVariables, asyncEnvironmentVariable],
[
...secrets,
...[

56
src/model/cloud-runner/workflows/build-automation-workflow.ts
View File

@ -1,26 +1,25 @@
import CloudRunnerLogger from '../services/cloud-runner-logger';
import { CloudRunnerFolders } from '../services/cloud-runner-folders';
import { CloudRunnerStepState } from '../cloud-runner-step-state';
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
import { CloudRunnerFolders } from '../options/cloud-runner-folders';
import { CloudRunnerStepParameters } from '../options/cloud-runner-step-parameters';
import { WorkflowInterface } from './workflow-interface';
import * as core from '@actions/core';
import { CloudRunnerCustomHooks } from '../services/cloud-runner-custom-hooks';
import { CommandHookService } from '../services/hooks/command-hook-service';
import path from 'node:path';
import CloudRunner from '../cloud-runner';
import CloudRunnerOptions from '../cloud-runner-options';
import { CloudRunnerCustomSteps } from '../services/cloud-runner-custom-steps';
import { ContainerHookService } from '../services/hooks/container-hook-service';
export class BuildAutomationWorkflow implements WorkflowInterface {
async run(cloudRunnerStepState: CloudRunnerStepState) {
async run(cloudRunnerStepState: CloudRunnerStepParameters) {
return await BuildAutomationWorkflow.standardBuildAutomation(cloudRunnerStepState.image, cloudRunnerStepState);
}
private static async standardBuildAutomation(baseImage: string, cloudRunnerStepState: CloudRunnerStepState) {
private static async standardBuildAutomation(baseImage: string, cloudRunnerStepState: CloudRunnerStepParameters) {
// TODO accept post and pre build steps as yaml files in the repo
CloudRunnerLogger.log(`Cloud Runner is running standard build automation`);
let output = '';
output += await CloudRunnerCustomSteps.RunPreBuildSteps(cloudRunnerStepState);
output += await ContainerHookService.RunPreBuildSteps(cloudRunnerStepState);
CloudRunnerLogger.logWithTime('Configurable pre build step(s) time');
if (!CloudRunner.buildParameters.isCliMode) core.startGroup('build');
@ -40,7 +39,7 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
CloudRunnerLogger.logWithTime('Build time');
output += await CloudRunnerCustomSteps.RunPostBuildSteps(cloudRunnerStepState);
output += await ContainerHookService.RunPostBuildSteps(cloudRunnerStepState);
CloudRunnerLogger.logWithTime('Configurable post build step(s) time');
CloudRunnerLogger.log(`Cloud Runner finished running standard build automation`);
@ -49,11 +48,11 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
}
private static get BuildWorkflow() {
const setupHooks = CloudRunnerCustomHooks.getHooks(CloudRunner.buildParameters.customJobHooks).filter((x) =>
x.step.includes(`setup`),
const setupHooks = CommandHookService.getHooks(CloudRunner.buildParameters.commandHooks).filter((x) =>
x.step?.includes(`setup`),
);
const buildHooks = CloudRunnerCustomHooks.getHooks(CloudRunner.buildParameters.customJobHooks).filter((x) =>
x.step.includes(`build`),
const buildHooks = CommandHookService.getHooks(CloudRunner.buildParameters.commandHooks).filter((x) =>
x.step?.includes(`build`),
);
const builderPath = CloudRunnerFolders.ToLinuxFolder(
path.join(CloudRunnerFolders.builderPathAbsolute, 'dist', `index.js`),
@ -65,16 +64,14 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
n 16.15.1 > /dev/null
npm --version
node --version
${BuildAutomationWorkflow.TreeCommand}
${setupHooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
export GITHUB_WORKSPACE="${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.repoPathAbsolute)}"
df -H /data/
${BuildAutomationWorkflow.setupCommands(builderPath)}
${setupHooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
${BuildAutomationWorkflow.TreeCommand}
${buildHooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
${BuildAutomationWorkflow.BuildCommands(builderPath)}
${buildHooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
${BuildAutomationWorkflow.TreeCommand}`;
${buildHooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}`;
}
private static setupCommands(builderPath: string) {
@ -84,24 +81,17 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
CloudRunnerFolders.unityBuilderRepoUrl
} "${CloudRunnerFolders.ToLinuxFolder(CloudRunnerFolders.builderPathAbsolute)}" && chmod +x ${builderPath}`;
const retainedWorkspaceCommands = `if [ -e "${CloudRunnerFolders.ToLinuxFolder(
CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute,
)}" ] && [ -e "${CloudRunnerFolders.ToLinuxFolder(
path.join(CloudRunnerFolders.repoPathAbsolute, `.git`),
)}" ]; then echo "Retained Workspace Already Exists!" ; fi`;
const cloneBuilderCommands = `if [ -e "${CloudRunnerFolders.ToLinuxFolder(
CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute,
)}" ] && [ -e "${CloudRunnerFolders.ToLinuxFolder(
path.join(CloudRunnerFolders.builderPathAbsolute, `.git`),
)}" ]; then echo "Builder Already Exists!"; else ${commands}; fi`;
)}" ] ; then echo "Builder Already Exists!" && tree ${
CloudRunnerFolders.builderPathAbsolute
}; else ${commands} ; fi`;
return `export GIT_DISCOVERY_ACROSS_FILESYSTEM=1
echo "downloading game-ci..."
${retainedWorkspaceCommands}
${cloneBuilderCommands}
echo "bootstrap game ci cloud runner..."
node ${builderPath} -m remote-cli-pre-build`;
${cloneBuilderCommands}
node ${builderPath} -m remote-cli-pre-build`;
}
private static BuildCommands(builderPath: string) {
@ -122,10 +112,4 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
chmod +x ${builderPath}
node ${builderPath} -m remote-cli-post-build`;
}
private static get TreeCommand(): string {
return CloudRunnerOptions.cloudRunnerDebugTree
? `tree -L 2 ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute} && tree -L 2 ${CloudRunnerFolders.cacheFolderForCacheKeyFull} && du -h -s /${CloudRunnerFolders.buildVolumeFolder}/ && du -h -s ${CloudRunnerFolders.cacheFolderForAllFull}`
: `#`;
}
}

26
src/model/cloud-runner/workflows/custom-workflow.ts
View File

@ -1,37 +1,37 @@
import CloudRunnerLogger from '../services/cloud-runner-logger';
import CloudRunnerSecret from '../services/cloud-runner-secret';
import { CloudRunnerFolders } from '../services/cloud-runner-folders';
import CloudRunnerEnvironmentVariable from '../services/cloud-runner-environment-variable';
import { CloudRunnerCustomSteps } from '../services/cloud-runner-custom-steps';
import { CustomStep } from '../services/custom-step';
import CloudRunnerLogger from '../services/core/cloud-runner-logger';
import CloudRunnerSecret from '../options/cloud-runner-secret';
import { CloudRunnerFolders } from '../options/cloud-runner-folders';
import CloudRunnerEnvironmentVariable from '../options/cloud-runner-environment-variable';
import { ContainerHookService } from '../services/hooks/container-hook-service';
import { ContainerHook } from '../services/hooks/container-hook';
import CloudRunner from '../cloud-runner';
export class CustomWorkflow {
public static async runCustomJobFromString(
public static async runContainerJobFromString(
buildSteps: string,
environmentVariables: CloudRunnerEnvironmentVariable[],
secrets: CloudRunnerSecret[],
): Promise<string> {
return await CustomWorkflow.runCustomJob(
CloudRunnerCustomSteps.ParseSteps(buildSteps),
return await CustomWorkflow.runContainerJob(
ContainerHookService.ParseContainerHooks(buildSteps),
environmentVariables,
secrets,
);
}
public static async runCustomJob(
buildSteps: CustomStep[],
public static async runContainerJob(
steps: ContainerHook[],
environmentVariables: CloudRunnerEnvironmentVariable[],
secrets: CloudRunnerSecret[],
) {
try {
CloudRunnerLogger.log(`Cloud Runner is running in custom job mode`);
let output = '';
// if (CloudRunner.buildParameters?.cloudRunnerDebug) {
// CloudRunnerLogger.log(`Custom Job Description \n${JSON.stringify(buildSteps, undefined, 4)}`);
// }
for (const step of buildSteps) {
for (const step of steps) {
CloudRunnerLogger.log(`Cloud Runner is running in custom job mode`);
output += await CloudRunner.Provider.runTaskInWorkflow(
CloudRunner.buildParameters.buildGuid,
step.image,

16
src/model/cloud-runner/workflows/workflow-composition-root.ts
View File

@ -1,20 +1,24 @@
import { CloudRunnerStepState } from '../cloud-runner-step-state';
import { CloudRunnerStepParameters } from '../options/cloud-runner-step-parameters';
import { CustomWorkflow } from './custom-workflow';
import { WorkflowInterface } from './workflow-interface';
import { BuildAutomationWorkflow } from './build-automation-workflow';
import CloudRunner from '../cloud-runner';
import CloudRunnerOptions from '../cloud-runner-options';
import CloudRunnerOptions from '../options/cloud-runner-options';
import { AsyncWorkflow } from './async-workflow';
export class WorkflowCompositionRoot implements WorkflowInterface {
async run(cloudRunnerStepState: CloudRunnerStepState) {
async run(cloudRunnerStepState: CloudRunnerStepParameters) {
try {
if (CloudRunnerOptions.asyncCloudRunner) {
if (
CloudRunnerOptions.asyncCloudRunner &&
!CloudRunner.isCloudRunnerAsyncEnvironment &&
!CloudRunner.isCloudRunnerEnvironment
) {
return await AsyncWorkflow.runAsyncWorkflow(cloudRunnerStepState.environment, cloudRunnerStepState.secrets);
}
if (CloudRunner.buildParameters.customJob !== '') {
return await CustomWorkflow.runCustomJobFromString(
return await CustomWorkflow.runContainerJobFromString(
CloudRunner.buildParameters.customJob,
cloudRunnerStepState.environment,
cloudRunnerStepState.secrets,
@ -22,7 +26,7 @@ export class WorkflowCompositionRoot implements WorkflowInterface {
}
return await new BuildAutomationWorkflow().run(
new CloudRunnerStepState(
new CloudRunnerStepParameters(
cloudRunnerStepState.image.toString(),
cloudRunnerStepState.environment,
cloudRunnerStepState.secrets,

4
src/model/cloud-runner/workflows/workflow-interface.ts
View File

@ -1,8 +1,8 @@
import { CloudRunnerStepState } from '../cloud-runner-step-state';
import { CloudRunnerStepParameters } from '../options/cloud-runner-step-parameters';
export interface WorkflowInterface {
run(
// eslint-disable-next-line no-unused-vars
cloudRunnerStepState: CloudRunnerStepState,
cloudRunnerStepState: CloudRunnerStepParameters,
): Promise<string>;
}

5
src/model/docker.ts
View File

@ -15,6 +15,7 @@ class Docker {
// eslint-disable-next-line unicorn/no-useless-undefined
options: ExecOptions | undefined = undefined,
entrypointBash: boolean = false,
errorWhenMissingUnityBuildResults: boolean = true,
) {
let runCommand = '';
switch (process.platform) {
@ -26,9 +27,9 @@ class Docker {
}
if (options) {
options.silent = silent;
await execWithErrorCheck(runCommand, undefined, options);
await execWithErrorCheck(runCommand, undefined, options, errorWhenMissingUnityBuildResults);
} else {
await execWithErrorCheck(runCommand, undefined, { silent });
await execWithErrorCheck(runCommand, undefined, { silent }, errorWhenMissingUnityBuildResults);
}
}

5
src/model/exec-with-error-check.ts
View File

@ -4,9 +4,14 @@ export async function execWithErrorCheck(
commandLine: string,
arguments_?: string[],
options?: ExecOptions,
errorWhenMissingUnityBuildResults: boolean = true,
): Promise<number> {
const result = await getExecOutput(commandLine, arguments_, options);
if (!errorWhenMissingUnityBuildResults) {
return result.exitCode;
}
// Check for errors in the Build Results section
const match = result.stdout.match(/^#\s*Build results\s*#(.*)^Size:/ms);

85
src/model/github.ts
View File

@ -1,6 +1,6 @@
import CloudRunnerLogger from './cloud-runner/services/cloud-runner-logger';
import CloudRunnerLogger from './cloud-runner/services/core/cloud-runner-logger';
import CloudRunner from './cloud-runner/cloud-runner';
import CloudRunnerOptions from './cloud-runner/cloud-runner-options';
import CloudRunnerOptions from './cloud-runner/options/cloud-runner-options';
import * as core from '@actions/core';
import { Octokit } from '@octokit/core';
@ -10,6 +10,7 @@ class GitHub {
private static longDescriptionContent: string = ``;
private static startedDate: string;
private static endedDate: string;
static result: string = ``;
private static get octokitDefaultToken() {
return new Octokit({
auth: process.env.GITHUB_TOKEN,
@ -33,7 +34,7 @@ class GitHub {
}
private static get checkRunId() {
return CloudRunner.githubCheckId;
return CloudRunner.buildParameters.githubCheckId;
}
private static get owner() {
@ -45,13 +46,12 @@ class GitHub {
}
public static async createGitHubCheck(summary: string) {
if (!CloudRunnerOptions.githubChecks) {
if (!CloudRunner.buildParameters.githubChecks) {
return ``;
}
GitHub.startedDate = new Date().toISOString();
CloudRunnerLogger.log(`POST /repos/${GitHub.owner}/${GitHub.repo}/check-runs`);
CloudRunnerLogger.log(`Creating inital github check`);
const data = {
owner: GitHub.owner,
repo: GitHub.repo,
@ -78,20 +78,27 @@ class GitHub {
};
const result = await GitHub.createGitHubCheckRequest(data);
return result.data.id;
return result.data.id.toString();
}
public static async updateGitHubCheck(
longDescription: string,
summary: any,
summary: string,
result = `neutral`,
status = `in_progress`,
) {
if (!CloudRunnerOptions.githubChecks) {
if (`${CloudRunner.buildParameters.githubChecks}` !== `true`) {
return;
}
CloudRunnerLogger.log(
`githubChecks: ${CloudRunner.buildParameters.githubChecks} checkRunId: ${GitHub.checkRunId} sha: ${GitHub.sha} async: ${CloudRunner.isCloudRunnerAsyncEnvironment}`,
);
GitHub.longDescriptionContent += `\n${longDescription}`;
if (GitHub.result !== `success` && GitHub.result !== `failure`) {
GitHub.result = result;
} else {
result = GitHub.result;
}
const data: any = {
owner: GitHub.owner,
repo: GitHub.repo,
@ -120,12 +127,9 @@ class GitHub {
data.conclusion = result;
}
if (await CloudRunnerOptions.asyncCloudRunner) {
await GitHub.runUpdateAsyncChecksWorkflow(data, `update`);
return;
}
await GitHub.updateGitHubCheckRequest(data);
await (CloudRunner.isCloudRunnerAsyncEnvironment
? GitHub.runUpdateAsyncChecksWorkflow(data, `update`)
: GitHub.updateGitHubCheckRequest(data));
}
public static async updateGitHubCheckRequest(data: any) {
@ -140,18 +144,16 @@ class GitHub {
if (mode === `create`) {
throw new Error(`Not supported: only use update`);
}
const workflowsResult = await GitHub.octokitDefaultToken.request(
`GET /repos/${GitHub.owner}/${GitHub.repo}/actions/workflows`,
{
owner: GitHub.owner,
repo: GitHub.repo,
},
);
const workflowsResult = await GitHub.octokitPAT.request(`GET /repos/{owner}/{repo}/actions/workflows`, {
owner: GitHub.owner,
repo: GitHub.repo,
});
const workflows = workflowsResult.data.workflows;
CloudRunnerLogger.log(`Got ${workflows.length} workflows`);
let selectedId = ``;
for (let index = 0; index < workflowsResult.data.total_count; index++) {
if (workflows[index].name === GitHub.asyncChecksApiWorkflowName) {
selectedId = workflows[index].id;
selectedId = workflows[index].id.toString();
}
}
if (selectedId === ``) {
@ -169,6 +171,41 @@ class GitHub {
},
});
}
static async triggerWorkflowOnComplete(triggerWorkflowOnComplete: string[]) {
const isLocalAsync = CloudRunner.buildParameters.asyncWorkflow && !CloudRunner.isCloudRunnerAsyncEnvironment;
if (isLocalAsync) {
return;
}
const workflowsResult = await GitHub.octokitPAT.request(`GET /repos/{owner}/{repo}/actions/workflows`, {
owner: GitHub.owner,
repo: GitHub.repo,
});
const workflows = workflowsResult.data.workflows;
CloudRunnerLogger.log(`Got ${workflows.length} workflows`);
for (const element of triggerWorkflowOnComplete) {
let selectedId = ``;
for (let index = 0; index < workflowsResult.data.total_count; index++) {
if (workflows[index].name === element) {
selectedId = workflows[index].id.toString();
}
}
if (selectedId === ``) {
core.info(JSON.stringify(workflows));
throw new Error(`no workflow with name "${GitHub.asyncChecksApiWorkflowName}"`);
}
await GitHub.octokitPAT.request(`POST /repos/{owner}/{repo}/actions/workflows/{workflow_id}/dispatches`, {
owner: GitHub.owner,
repo: GitHub.repo,
// eslint-disable-next-line camelcase
workflow_id: selectedId,
ref: CloudRunnerOptions.branch,
inputs: {
buildGuid: CloudRunner.buildParameters.buildGuid,
},
});
}
}
}
export default GitHub;

2
src/model/image-environment-factory.ts
View File

@ -65,7 +65,7 @@ class ImageEnvironmentFactory {
{ name: 'RUNNER_TEMP', value: process.env.RUNNER_TEMP },
{ name: 'RUNNER_WORKSPACE', value: process.env.RUNNER_WORKSPACE },
];
if (parameters.cloudRunnerCluster === 'local-docker') {
if (parameters.providerStrategy === 'local-docker') {
for (const element of additionalVariables) {
if (
environmentVariables.find(

6
src/model/input-readers/generic-input-reader.ts
View File

@ -1,9 +1,9 @@
import { CloudRunnerSystem } from '../cloud-runner/services/cloud-runner-system';
import CloudRunnerOptions from '../cloud-runner/cloud-runner-options';
import { CloudRunnerSystem } from '../cloud-runner/services/core/cloud-runner-system';
import CloudRunnerOptions from '../cloud-runner/options/cloud-runner-options';
export class GenericInputReader {
public static async Run(command: string) {
if (CloudRunnerOptions.cloudRunnerCluster === 'local') {
if (CloudRunnerOptions.providerStrategy === 'local') {
return '';
}

8
src/model/input-readers/git-repo.test.ts
View File

@ -1,6 +1,6 @@
import { GitRepoReader } from './git-repo';
import { CloudRunnerSystem } from '../cloud-runner/services/cloud-runner-system';
import CloudRunnerOptions from '../cloud-runner/cloud-runner-options';
import { CloudRunnerSystem } from '../cloud-runner/services/core/cloud-runner-system';
import CloudRunnerOptions from '../cloud-runner/options/cloud-runner-options';
describe(`git repo tests`, () => {
it(`Branch value parsed from CLI to not contain illegal characters`, async () => {
@ -11,14 +11,14 @@ describe(`git repo tests`, () => {
it(`returns valid branch name when using https`, async () => {
const mockValue = 'https://github.com/example/example.git';
await jest.spyOn(CloudRunnerSystem, 'Run').mockReturnValue(Promise.resolve(mockValue));
await jest.spyOn(CloudRunnerOptions, 'cloudRunnerCluster', 'get').mockReturnValue('not-local');
await jest.spyOn(CloudRunnerOptions, 'providerStrategy', 'get').mockReturnValue('not-local');
expect(await GitRepoReader.GetRemote()).toEqual(`example/example`);
});
it(`returns valid branch name when using ssh`, async () => {
const mockValue = 'git@github.com:example/example.git';
await jest.spyOn(CloudRunnerSystem, 'Run').mockReturnValue(Promise.resolve(mockValue));
await jest.spyOn(CloudRunnerOptions, 'cloudRunnerCluster', 'get').mockReturnValue('not-local');
await jest.spyOn(CloudRunnerOptions, 'providerStrategy', 'get').mockReturnValue('not-local');
expect(await GitRepoReader.GetRemote()).toEqual(`example/example`);
});
});

10
src/model/input-readers/git-repo.ts
View File

@ -1,13 +1,13 @@
import { assert } from 'node:console';
import fs from 'node:fs';
import { CloudRunnerSystem } from '../cloud-runner/services/cloud-runner-system';
import CloudRunnerLogger from '../cloud-runner/services/cloud-runner-logger';
import CloudRunnerOptions from '../cloud-runner/cloud-runner-options';
import { CloudRunnerSystem } from '../cloud-runner/services/core/cloud-runner-system';
import CloudRunnerLogger from '../cloud-runner/services/core/cloud-runner-logger';
import CloudRunnerOptions from '../cloud-runner/options/cloud-runner-options';
import Input from '../input';
export class GitRepoReader {
public static async GetRemote() {
if (CloudRunnerOptions.cloudRunnerCluster === 'local') {
if (CloudRunnerOptions.providerStrategy === 'local') {
return '';
}
assert(fs.existsSync(`.git`));
@ -22,7 +22,7 @@ export class GitRepoReader {
}
public static async GetBranch() {
if (CloudRunnerOptions.cloudRunnerCluster === 'local') {
if (CloudRunnerOptions.providerStrategy === 'local') {
return '';
}
assert(fs.existsSync(`.git`));

6
src/model/input-readers/github-cli.ts
View File

@ -1,10 +1,10 @@
import { CloudRunnerSystem } from '../cloud-runner/services/cloud-runner-system';
import { CloudRunnerSystem } from '../cloud-runner/services/core/cloud-runner-system';
import * as core from '@actions/core';
import CloudRunnerOptions from '../cloud-runner/cloud-runner-options';
import CloudRunnerOptions from '../cloud-runner/options/cloud-runner-options';
export class GithubCliReader {
static async GetGitHubAuthToken() {
if (CloudRunnerOptions.cloudRunnerCluster === 'local') {
if (CloudRunnerOptions.providerStrategy === 'local') {
return '';
}
try {

4
src/model/input-readers/test-license-reader.ts
View File

@ -1,10 +1,10 @@
import path from 'node:path';
import fs from 'node:fs';
import YAML from 'yaml';
import CloudRunnerOptions from '../cloud-runner/cloud-runner-options';
import CloudRunnerOptions from '../cloud-runner/options/cloud-runner-options';
export function ReadLicense(): string {
if (CloudRunnerOptions.cloudRunnerCluster === 'local') {
if (CloudRunnerOptions.providerStrategy === 'local') {
return '';
}
const pipelineFile = path.join(__dirname, `.github`, `workflows`, `cloud-runner-k8s-pipeline.yml`);

2
src/model/input.ts
View File

@ -1,7 +1,7 @@
import fs from 'node:fs';
import path from 'node:path';
import { Cli } from './cli/cli';
import CloudRunnerQueryOverride from './cloud-runner/services/cloud-runner-query-override';
import CloudRunnerQueryOverride from './cloud-runner/options/cloud-runner-query-override';
import Platform from './platform';
import GitHub from './github';

5
yarn.lock
View File

@ -5946,6 +5946,11 @@ uuid@^8.3.0, uuid@^8.3.2:
resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2"
integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==
uuid@^9.0.0:
version "9.0.0"
resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.0.tgz#592f550650024a38ceb0c562f2f6aa435761efb5"
integrity sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==
v8-compile-cache@^2.0.3:
version "2.3.0"
resolved "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz"