mirror of
https://github.com/game-ci/unity-builder.git
synced 2025-07-04 12:25:19 -04:00
Cloud Runner v0 - Reliable and trimmed down cloud runner (#353)
* Update cloud-runner-aws-pipeline.yml * Update cloud-runner-k8s-pipeline.yml * yarn build * yarn build * correct branch ref * correct branch ref passed to target repo * Create k8s-tests.yml * Delete k8s-tests.yml * correct branch ref passed to target repo * correct branch ref passed to target repo * Always describe AWS tasks for now, because unstable error handling * Remove unused tree commands * Use lfs guid sum * Simple override cache push * Simple override cache push and pull override to allow pure cloud storage driven caching * Removal of early branch (breaks lfs caching) * Remove unused tree commands * Update action.yml * Update action.yml * Support cache and input override commands as input + full support custom hooks * Increase k8s timeout * replace filename being appended for unknclear reason * cache key should not contain whitespaces * Always try and deploy rook for k8s * Apply k8s files for rook * Update action.yml * Apply k8s files for rook * Apply k8s files for rook * cache test and action description for kuber storage class * Correct test and implement dependency health check and start * GCP-secret run, cache key * lfs smudge set explicit and undo explicit * Run using external secret provider to speed up input * Update cloud-runner-aws-pipeline.yml * Add nodejs as build step dependency * Add nodejs as build step dependency * Cloud Runner Tests must be specified to capture logs from cloud runner for tests * Cloud Runner Tests must be specified to capture logs from cloud runner for tests * Refactor and cleanup - no async input, combined setup/build, removed github logs for cli runs * Refactor and cleanup - no async input, combined setup/build, removed github logs for cli runs * Refactor and cleanup - no async input, combined setup/build, removed github logs for cli runs * Refactor and cleanup - no async input, combined setup/build, removed github logs for cli runs * Refactor and cleanup - no async input, combined setup/build, removed github logs for cli runs * better defaults for new inputs * better defaults * merge latest * force build update * use npm n to update node in unity builder * use npm n to update node in unity builder * use npm n to update node in unity builder * correct new line * quiet zipping * quiet zipping * default secrets for unity username and password * default secrets for unity username and password * ls active directory before lfs install * Get cloud runner secrets from * Get cloud runner secrets from * Cleanup setup of default secrets * Various fixes * Cleanup setup of default secrets * Various fixes * Various fixes * Various fixes * Various fixes * Various fixes * Various fixes * Various fixes * Various fixes * Various fixes * Various fixes * Various fixes * Various fixes * Various fixes * Various fixes * AWS secrets manager support * less caching logs * default k8s storage class to pd-standard * more readable build commands * Capture aws exit code 1 reliably * Always replace /head from branch * k8s default storage class to standard-rwo * cleanup * further cleanup input * further cleanup input * further cleanup input * further cleanup input * further cleanup input * folder sizes to inspect caching * dir command for local cloud runner test * k8s wait for pending because pvc will not create earlier * prefer k8s standard storage * handle empty string as cloud runner cluster input * local-system is now used for cloud runner test implementation AND correctly unset test CLI input * local-system is now used for cloud runner test implementation AND correctly unset test CLI input * fix unterminated quote * fix unterminated quote * do not share build parameters in tests - in cloud runner this will cause conflicts with resouces of the same name * remove head and heads from branch prefix * fix reversed caching direction of cache-push * fixes * fixes * fixes * cachePull cli * fixes * fixes * fixes * fixes * fixes * order cache test to be first * order cache test to be first * fixes * populate cache key instead of using branch * cleanup cli * garbage-collect-aws cli can iterate over aws resources and cli scans all ts files * import cli methods * import cli files explicitly * import cli files explicitly * import cli files explicitly * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * import cli methods * log parameters in cloud runner parameter test * log parameters in cloud runner parameter test * log parameters in cloud runner parameter test * Cloud runner param test before caching because we have a fast local cache test now * Using custom build path relative to repo root rather than project root * aws-garbage-collect at end of pipeline * aws-garbage-collect do not actually delete anything for now - just list * remove some legacy du commands * Update cloud-runner-aws-pipeline.yml * log contents after cache pull and fix some scenarios with duplicate secrets * log contents after cache pull and fix some scenarios with duplicate secrets * log contents after cache pull and fix some scenarios with duplicate secrets * PR comments * Replace guid with uuid package * use fileExists lambda instead of stat to check file exists in caching * build failed results in core error message * Delete sample.txt
This commit is contained in:
Frostebite
GitHub
parent
2b399b2641
commit
a61c02481f
11
.github/workflows/cloud-runner-aws-pipeline.yml
vendored
11
.github/workflows/cloud-runner-aws-pipeline.yml
vendored
@ -61,12 +61,15 @@ jobs:
|
||||
aws-region: eu-west-2
|
||||
- run: yarn
|
||||
- run: yarn run cli --help
|
||||
- run: yarn run test "caching"
|
||||
- run: yarn run test-i-aws
|
||||
env:
|
||||
UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
|
||||
PROJECT_PATH: ${{ matrix.projectPath }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
TARGET_PLATFORM: ${{ matrix.targetPlatform }}
|
||||
cloudRunnerTests: true
|
||||
versioning: None
|
||||
- uses: ./
|
||||
id: aws-fargate-unity-build
|
||||
timeout-minutes: 25
|
||||
@ -87,9 +90,8 @@ jobs:
|
||||
aws s3 ls
|
||||
aws s3 ls game-ci-test-storage
|
||||
ls /data/cache/$CACHE_KEY
|
||||
echo "/data/cache/$CACHE_KEY/build-$BUILD_GUID.zip s3://game-ci-test-storage/$CACHE_KEY/$BUILD_FILE"
|
||||
aws s3 cp /data/cache/$CACHE_KEY/build-$BUILD_GUID.zip s3://game-ci-test-storage/$CACHE_KEY/build-$BUILD_GUID.zip
|
||||
aws s3 cp /data/cache/$CACHE_KEY s3://game-ci-test-storage/$CACHE_KEY/$BUILD_GUID
|
||||
ls /data/cache/$CACHE_KEY/build
|
||||
aws s3 cp /data/cache/$CACHE_KEY/build/build-$BUILD_GUID.zip s3://game-ci-test-storage/$CACHE_KEY/build-$BUILD_GUID.zip
|
||||
secrets:
|
||||
- name: awsAccessKeyId
|
||||
value: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
@ -98,8 +100,9 @@ jobs:
|
||||
- name: awsDefaultRegion
|
||||
value: eu-west-2
|
||||
- run: |
|
||||
aws s3 cp s3://game-ci-test-storage/${{ steps.aws-fargate-unity-build.outputs.BRANCH }}/build-${{ steps.aws-fargate-unity-build.outputs.BUILD_GUID }}.zip build-${{ steps.aws-fargate-unity-build.outputs.BUILD_GUID }}.zip
|
||||
aws s3 cp s3://game-ci-test-storage/${{ steps.aws-fargate-unity-build.outputs.CACHE_KEY }}/build-${{ steps.aws-fargate-unity-build.outputs.BUILD_GUID }}.zip build-${{ steps.aws-fargate-unity-build.outputs.BUILD_GUID }}.zip
|
||||
ls
|
||||
- run: yarn run cli -m aws-garbage-collect
|
||||
###########################
|
||||
# Upload #
|
||||
###########################
|
||||
|
||||
13
.github/workflows/cloud-runner-k8s-pipeline.yml
vendored
13
.github/workflows/cloud-runner-k8s-pipeline.yml
vendored
@ -50,7 +50,7 @@ jobs:
|
||||
###########################
|
||||
# Setup #
|
||||
###########################
|
||||
- uses: google-github-actions/setup-gcloud@master
|
||||
- uses: google-github-actions/setup-gcloud@v0
|
||||
with:
|
||||
version: '288.0.0'
|
||||
service_account_email: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_EMAIL }}
|
||||
@ -66,6 +66,7 @@ jobs:
|
||||
node-version: 12.x
|
||||
- run: yarn
|
||||
- run: yarn run cli --help
|
||||
- run: yarn run test "caching"
|
||||
- name: Cloud Runner Test Suite
|
||||
run: yarn run test-i-k8s --detectOpenHandles --forceExit
|
||||
env:
|
||||
@ -75,6 +76,8 @@ jobs:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
KUBE_CONFIG: ${{ steps.read-base64.outputs.base64 }}
|
||||
unityVersion: ${{ matrix.unityVersion }}
|
||||
cloudRunnerTests: true
|
||||
versioning: None
|
||||
|
||||
###########################
|
||||
# Cloud Runner Build Test #
|
||||
@ -101,9 +104,9 @@ jobs:
|
||||
aws configure set region $AWS_DEFAULT_REGION --profile default
|
||||
aws s3 ls
|
||||
aws s3 ls game-ci-test-storage
|
||||
ls /data/cache/$BRANCH
|
||||
echo "/data/cache/$BRANCH/build-$BUILD_GUID.zip s3://game-ci-test-storage/$BRANCH/$BUILD_FILE"
|
||||
aws s3 cp /data/cache/$BRANCH/build-$BUILD_GUID.zip s3://game-ci-test-storage/$BRANCH/build-$BUILD_GUID.zip
|
||||
ls /data/cache/$CACHE_KEY
|
||||
echo "/data/cache/$CACHE_KEY/build/build-$BUILD_GUID.zip s3://game-ci-test-storage/$CACHE_KEY/$BUILD_FILE"
|
||||
aws s3 cp /data/cache/$CACHE_KEY/build/build-$BUILD_GUID.zip s3://game-ci-test-storage/$CACHE_KEY/build-$BUILD_GUID.zip
|
||||
secrets:
|
||||
- name: awsAccessKeyId
|
||||
value: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
@ -112,7 +115,7 @@ jobs:
|
||||
- name: awsDefaultRegion
|
||||
value: eu-west-2
|
||||
- run: |
|
||||
aws s3 cp s3://game-ci-test-storage/${{ steps.k8s-unity-build.outputs.BRANCH }}/build-${{ steps.k8s-unity-build.outputs.BUILD_GUID }}.zip build-${{ steps.k8s-unity-build.outputs.BUILD_GUID }}.zip
|
||||
aws s3 cp s3://game-ci-test-storage/${{ steps.k8s-unity-build.outputs.CACHE_KEY }}/build-${{ steps.k8s-unity-build.outputs.BUILD_GUID }}.zip build-${{ steps.k8s-unity-build.outputs.BUILD_GUID }}.zip
|
||||
ls
|
||||
###########################
|
||||
# Upload #
|
||||
|
||||
36
action.yml
36
action.yml
@ -98,6 +98,10 @@ inputs:
|
||||
required: false
|
||||
default: ''
|
||||
description: 'Run a pre build job after the repository setup but before the build job (in yaml format with the keys image, secrets (name, value object array), command line string)'
|
||||
customJobHooks:
|
||||
required: false
|
||||
default: ''
|
||||
description: 'Specify custom commands and trigger hooks (injects commands into jobs)'
|
||||
customJob:
|
||||
required: false
|
||||
default: ''
|
||||
@ -118,10 +122,22 @@ inputs:
|
||||
default: '750M'
|
||||
required: false
|
||||
description: 'Amount of memory to assign the remote build container'
|
||||
githubToken:
|
||||
cachePushOverrideCommand:
|
||||
default: ''
|
||||
required: false
|
||||
description: 'GitHub token for cloning, only needed when kubeconfig is used.'
|
||||
description: 'A command run every time a file is pushed to cache, formatted with input file path and remote cache path'
|
||||
cachePullOverrideCommand:
|
||||
default: ''
|
||||
required: false
|
||||
description: 'A command run every time before a file is being pulled from cache, formatted with request cache file and destination path'
|
||||
readInputFromOverrideList:
|
||||
default: ''
|
||||
required: false
|
||||
description: 'Comma separated list of input value names to read from "input override command"'
|
||||
readInputOverrideCommand:
|
||||
default: ''
|
||||
required: false
|
||||
description: 'Extend game ci by specifying a command to execute to pull input from external source e.g cloud provider secret managers'
|
||||
kubeConfig:
|
||||
default: ''
|
||||
required: false
|
||||
@ -130,10 +146,26 @@ inputs:
|
||||
default: ''
|
||||
required: false
|
||||
description: 'Supply a Persistent Volume Claim name to use for the Unity build.'
|
||||
kubeStorageClass:
|
||||
default: ''
|
||||
required: false
|
||||
description: 'Kubernetes storage class to use for cloud runner jobs, leave empty to install rook cluster.'
|
||||
kubeVolumeSize:
|
||||
default: '5Gi'
|
||||
required: false
|
||||
description: 'Amount of disc space to assign the Kubernetes Persistent Volume'
|
||||
cacheKey:
|
||||
default: ''
|
||||
required: false
|
||||
description: 'Cache key to indicate bucket for cache'
|
||||
checkDependencyHealthOverride:
|
||||
default: ''
|
||||
required: false
|
||||
description: 'Use to specify a way to check depdency services health to enable resilient self-starting jobs'
|
||||
startDependenciesOverride:
|
||||
default: ''
|
||||
required: false
|
||||
description: 'Use to specify a way to start depdency services health to enable resilient self-starting jobs'
|
||||
outputs:
|
||||
volume:
|
||||
description: 'The Persistent Volume (PV) where the build artifacts have been stored by Kubernetes'
|
||||
|
||||
BIN
dist/index.js
generated
vendored
BIN
dist/index.js
generated
vendored
Binary file not shown.
BIN
dist/index.js.map
generated
vendored
BIN
dist/index.js.map
generated
vendored
Binary file not shown.
6956
package-lock.json
generated
Normal file
6956
package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
@ -12,6 +12,9 @@
|
||||
"lint": "prettier --check \"src/**/*.{js,ts}\" && eslint src/**/*.ts",
|
||||
"format": "prettier --write \"src/**/*.{js,ts}\"",
|
||||
"cli": "yarn ts-node src/index.ts -m cli",
|
||||
"gcp-secrets-tests": "cross-env cloudRunnerCluster=aws cloudRunnerTests=true readInputOverrideCommand=\"gcloud secrets versions access 1 --secret=\"{0}\"\" populateOverride=true readInputFromOverrideList=UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD yarn test -i -t \"cloud runner\"",
|
||||
"gcp-secrets-cli": "cross-env cloudRunnerTests=true readInputOverrideCommand=\"gcloud secrets versions access 1 --secret=\"{0}\"\" yarn ts-node src/index.ts -m cli --populateOverride true --readInputFromOverrideList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD",
|
||||
"aws-secrets-cli": "cross-env cloudRunnerTests=true readInputOverrideCommand=\"aws secretsmanager get-secret-value --secret-id {0}\" yarn ts-node src/index.ts -m cli --populateOverride true --readInputFromOverrideList UNITY_EMAIL,UNITY_SERIAL,UNITY_PASSWORD",
|
||||
"cli-aws": "cross-env cloudRunnerCluster=aws yarn run test-cli",
|
||||
"cli-k8s": "cross-env cloudRunnerCluster=k8s yarn run test-cli",
|
||||
"test-cli": "cross-env cloudRunnerTests=true yarn ts-node src/index.ts -m cli --projectPath test-project",
|
||||
@ -37,12 +40,13 @@
|
||||
"reflect-metadata": "^0.1.13",
|
||||
"semver": "^7.3.5",
|
||||
"unity-changeset": "^1.6.0",
|
||||
"uuid": "^8.3.2",
|
||||
"yaml": "^1.10.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@arkweid/lefthook": "^0.7.7",
|
||||
"@types/jest": "^27.4.1",
|
||||
"@types/node": "^17.0.21",
|
||||
"@types/node": "^17.0.23",
|
||||
"@types/semver": "^7.3.9",
|
||||
"@typescript-eslint/parser": "4.8.1",
|
||||
"@vercel/ncc": "^0.33.3",
|
||||
|
||||
19
src/index.ts
19
src/index.ts
@ -1,10 +1,14 @@
|
||||
import * as core from '@actions/core';
|
||||
import { Action, BuildParameters, Cache, Docker, ImageTag, Output, CloudRunner } from './model';
|
||||
import { CLI } from './model/cli/cli';
|
||||
import { Cli } from './model/cli/cli';
|
||||
import MacBuilder from './model/mac-builder';
|
||||
import PlatformSetup from './model/platform-setup';
|
||||
async function runMain() {
|
||||
try {
|
||||
if (Cli.InitCliMode()) {
|
||||
await Cli.RunCli();
|
||||
return;
|
||||
}
|
||||
Action.checkCompatibility();
|
||||
Cache.verify();
|
||||
|
||||
@ -13,11 +17,7 @@ async function runMain() {
|
||||
const buildParameters = await BuildParameters.create();
|
||||
const baseImage = new ImageTag(buildParameters);
|
||||
|
||||
if (
|
||||
buildParameters.cloudRunnerCluster &&
|
||||
buildParameters.cloudRunnerCluster !== '' &&
|
||||
buildParameters.cloudRunnerCluster !== 'local'
|
||||
) {
|
||||
if (buildParameters.cloudRunnerCluster !== 'local') {
|
||||
await CloudRunner.run(buildParameters, baseImage.toString());
|
||||
} else {
|
||||
core.info('Building locally');
|
||||
@ -35,9 +35,4 @@ async function runMain() {
|
||||
core.setFailed((error as Error).message);
|
||||
}
|
||||
}
|
||||
const options = CLI.SetupCli();
|
||||
if (CLI.isCliMode(options)) {
|
||||
CLI.RunCli(options);
|
||||
} else {
|
||||
runMain();
|
||||
}
|
||||
runMain();
|
||||
|
||||
@ -1,12 +1,14 @@
|
||||
import { customAlphabet } from 'nanoid';
|
||||
import * as core from '@actions/core';
|
||||
import AndroidVersioning from './android-versioning';
|
||||
import CloudRunnerConstants from './cloud-runner/services/cloud-runner-constants';
|
||||
import CloudRunnerNamespace from './cloud-runner/services/cloud-runner-namespace';
|
||||
import CloudRunnerBuildGuid from './cloud-runner/services/cloud-runner-guid';
|
||||
import Input from './input';
|
||||
import Platform from './platform';
|
||||
import UnityVersioning from './unity-versioning';
|
||||
import Versioning from './versioning';
|
||||
import { GitRepoReader } from './input-readers/git-repo';
|
||||
import { GithubCliReader } from './input-readers/github-cli';
|
||||
import { Cli } from './cli/cli';
|
||||
|
||||
class BuildParameters {
|
||||
public editorVersion!: string;
|
||||
@ -33,15 +35,22 @@ class BuildParameters {
|
||||
public cloudRunnerCluster!: string;
|
||||
public awsBaseStackName!: string;
|
||||
public gitPrivateToken!: string;
|
||||
public remoteBuildCluster!: string;
|
||||
public awsStackName!: string;
|
||||
public kubeConfig!: string;
|
||||
public githubToken!: string;
|
||||
public cloudRunnerMemory!: string;
|
||||
public cloudRunnerCpu!: string;
|
||||
public kubeVolumeSize!: string;
|
||||
public kubeVolume!: string;
|
||||
public kubeStorageClass!: string;
|
||||
public chownFilesTo!: string;
|
||||
public customJobHooks!: string;
|
||||
public cachePushOverrideCommand!: string;
|
||||
public cachePullOverrideCommand!: string;
|
||||
public readInputFromOverrideList!: string;
|
||||
public readInputOverrideCommand!: string;
|
||||
public checkDependencyHealthOverride!: string;
|
||||
public startDependenciesOverride!: string;
|
||||
public cacheKey!: string;
|
||||
|
||||
public postBuildSteps!: string;
|
||||
public preBuildSteps!: string;
|
||||
@ -52,6 +61,10 @@ class BuildParameters {
|
||||
public gitSha!: string;
|
||||
public logId!: string;
|
||||
public buildGuid!: string;
|
||||
public cloudRunnerBranch!: string;
|
||||
public cloudRunnerIntegrationTests!: boolean;
|
||||
public cloudRunnerBuilderPlatform!: string | undefined;
|
||||
public isCliMode!: boolean;
|
||||
|
||||
static async create(): Promise<BuildParameters> {
|
||||
const buildFile = this.parseBuildFile(Input.buildName, Input.targetPlatform, Input.androidAppBundle);
|
||||
@ -63,7 +76,7 @@ class BuildParameters {
|
||||
// Todo - Don't use process.env directly, that's what the input model class is for.
|
||||
// ---
|
||||
let unitySerial = '';
|
||||
if (!process.env.UNITY_SERIAL) {
|
||||
if (!process.env.UNITY_SERIAL && Input.githubInputEnabled && Cli.options === undefined) {
|
||||
//No serial was present so it is a personal license that we need to convert
|
||||
if (!process.env.UNITY_LICENSE) {
|
||||
throw new Error(`Missing Unity License File and no Serial was found. If this
|
||||
@ -75,8 +88,6 @@ class BuildParameters {
|
||||
} else {
|
||||
unitySerial = process.env.UNITY_SERIAL!;
|
||||
}
|
||||
core.setSecret(unitySerial);
|
||||
// ---
|
||||
|
||||
return {
|
||||
editorVersion,
|
||||
@ -101,12 +112,12 @@ class BuildParameters {
|
||||
androidSdkManagerParameters,
|
||||
customParameters: Input.customParameters,
|
||||
sshAgent: Input.sshAgent,
|
||||
gitPrivateToken: await Input.gitPrivateToken(),
|
||||
gitPrivateToken: Input.gitPrivateToken || (await GithubCliReader.GetGitHubAuthToken()),
|
||||
chownFilesTo: Input.chownFilesTo,
|
||||
cloudRunnerCluster: Input.cloudRunnerCluster,
|
||||
cloudRunnerBuilderPlatform: Input.cloudRunnerBuilderPlatform,
|
||||
awsBaseStackName: Input.awsBaseStackName,
|
||||
kubeConfig: Input.kubeConfig,
|
||||
githubToken: await Input.githubToken(),
|
||||
cloudRunnerMemory: Input.cloudRunnerMemory,
|
||||
cloudRunnerCpu: Input.cloudRunnerCpu,
|
||||
kubeVolumeSize: Input.kubeVolumeSize,
|
||||
@ -115,14 +126,24 @@ class BuildParameters {
|
||||
preBuildSteps: Input.preBuildSteps,
|
||||
customJob: Input.customJob,
|
||||
runNumber: Input.runNumber,
|
||||
branch: await Input.branch(),
|
||||
// Todo - move this out of UserInput and into some class that determines additional information (as needed)
|
||||
githubRepo: await Input.githubRepo(),
|
||||
remoteBuildCluster: Input.cloudRunnerCluster,
|
||||
branch: Input.branch.replace('/head', '') || (await GitRepoReader.GetBranch()),
|
||||
cloudRunnerBranch: Input.cloudRunnerBranch.split('/').reverse()[0],
|
||||
cloudRunnerIntegrationTests: Input.cloudRunnerTests,
|
||||
githubRepo: Input.githubRepo || (await GitRepoReader.GetRemote()) || 'game-ci/unity-builder',
|
||||
isCliMode: Cli.isCliMode,
|
||||
awsStackName: Input.awsBaseStackName,
|
||||
gitSha: Input.gitSha,
|
||||
logId: customAlphabet(CloudRunnerConstants.alphabet, 9)(),
|
||||
buildGuid: CloudRunnerNamespace.generateBuildName(Input.runNumber, Input.targetPlatform),
|
||||
buildGuid: CloudRunnerBuildGuid.generateGuid(Input.runNumber, Input.targetPlatform),
|
||||
customJobHooks: Input.customJobHooks(),
|
||||
cachePullOverrideCommand: Input.cachePullOverrideCommand(),
|
||||
cachePushOverrideCommand: Input.cachePushOverrideCommand(),
|
||||
readInputOverrideCommand: Input.readInputOverrideCommand(),
|
||||
readInputFromOverrideList: Input.readInputFromOverrideList(),
|
||||
kubeStorageClass: Input.kubeStorageClass,
|
||||
checkDependencyHealthOverride: Input.checkDependencyHealthOverride,
|
||||
startDependenciesOverride: Input.startDependenciesOverride,
|
||||
cacheKey: Input.cacheKey,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@ -1,23 +0,0 @@
|
||||
const targets = new Array();
|
||||
export function CliFunction(key: string, description: string) {
|
||||
return function (target: any, propertyKey: string, descriptor: PropertyDescriptor) {
|
||||
targets.push({
|
||||
target,
|
||||
propertyKey,
|
||||
descriptor,
|
||||
key,
|
||||
description,
|
||||
});
|
||||
};
|
||||
}
|
||||
export function GetCliFunctions(key) {
|
||||
return targets.find((x) => x.key === key);
|
||||
}
|
||||
export function GetAllCliModes() {
|
||||
return targets.map((x) => {
|
||||
return {
|
||||
key: x.key,
|
||||
description: x.description,
|
||||
};
|
||||
});
|
||||
}
|
||||
44
src/model/cli/cli-functions-repository.ts
Normal file
44
src/model/cli/cli-functions-repository.ts
Normal file
@ -0,0 +1,44 @@
|
||||
export class CliFunctionsRepository {
|
||||
private static targets: any[] = [];
|
||||
public static PushCliFunction(
|
||||
target: any,
|
||||
propertyKey: string,
|
||||
descriptor: PropertyDescriptor,
|
||||
key: string,
|
||||
description: string,
|
||||
) {
|
||||
CliFunctionsRepository.targets.push({
|
||||
target,
|
||||
propertyKey,
|
||||
descriptor,
|
||||
key,
|
||||
description,
|
||||
});
|
||||
}
|
||||
|
||||
public static GetCliFunctions(key) {
|
||||
const results = CliFunctionsRepository.targets.find((x) => x.key === key);
|
||||
if (results === undefined || results.length === 0) {
|
||||
throw new Error(`no CLI mode found for ${key}`);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
public static GetAllCliModes() {
|
||||
return CliFunctionsRepository.targets.map((x) => {
|
||||
return {
|
||||
key: x.key,
|
||||
description: x.description,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
public static PushCliFunctionSource(cliFunction: any) {}
|
||||
}
|
||||
|
||||
export function CliFunction(key: string, description: string) {
|
||||
return (target: any, propertyKey: string, descriptor: PropertyDescriptor) => {
|
||||
CliFunctionsRepository.PushCliFunction(target, propertyKey, descriptor, key, description);
|
||||
};
|
||||
}
|
||||
@ -3,55 +3,85 @@ import { BuildParameters, CloudRunner, ImageTag, Input } from '..';
|
||||
import * as core from '@actions/core';
|
||||
import { ActionYamlReader } from '../input-readers/action-yaml';
|
||||
import CloudRunnerLogger from '../cloud-runner/services/cloud-runner-logger';
|
||||
import { CliFunction, GetAllCliModes, GetCliFunctions } from './cli-decorator';
|
||||
import { RemoteClientLogger } from './remote-client/remote-client-services/remote-client-logger';
|
||||
import { CloudRunnerState } from '../cloud-runner/state/cloud-runner-state';
|
||||
import { SetupCloudRunnerRepository } from './remote-client/setup-cloud-runner-repository';
|
||||
import * as SDK from 'aws-sdk';
|
||||
import CloudRunnerQueryOverride from '../cloud-runner/services/cloud-runner-query-override';
|
||||
import { CliFunction, CliFunctionsRepository } from './cli-functions-repository';
|
||||
import { AwsCliCommands } from '../cloud-runner/providers/aws/commands/aws-cli-commands';
|
||||
import { Caching } from '../cloud-runner/remote-client/caching';
|
||||
import { LfsHashing } from '../cloud-runner/services/lfs-hashing';
|
||||
import { RemoteClient } from '../cloud-runner/remote-client';
|
||||
|
||||
export class CLI {
|
||||
static async RunCli(options: any): Promise<void> {
|
||||
Input.githubInputEnabled = false;
|
||||
|
||||
const results = GetCliFunctions(options.mode);
|
||||
|
||||
if (results === undefined || results.length === 0) {
|
||||
throw new Error('no CLI mode found');
|
||||
export class Cli {
|
||||
public static options;
|
||||
static get isCliMode() {
|
||||
return Cli.options !== undefined && Cli.options.mode !== undefined && Cli.options.mode !== '';
|
||||
}
|
||||
public static query(key, alternativeKey) {
|
||||
if (Cli.options && Cli.options[key] !== undefined) {
|
||||
return Cli.options[key];
|
||||
}
|
||||
|
||||
CloudRunnerLogger.log(`Entrypoint: ${results.key}`);
|
||||
|
||||
options.versioning = 'None';
|
||||
Input.cliOptions = options;
|
||||
return await results.target[results.propertyKey]();
|
||||
}
|
||||
static isCliMode(options: any) {
|
||||
return options.mode !== undefined && options.mode !== '';
|
||||
if (Cli.options && alternativeKey && Cli.options[alternativeKey] !== undefined) {
|
||||
return Cli.options[alternativeKey];
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
public static SetupCli() {
|
||||
public static InitCliMode() {
|
||||
CliFunctionsRepository.PushCliFunctionSource(AwsCliCommands);
|
||||
CliFunctionsRepository.PushCliFunctionSource(Caching);
|
||||
CliFunctionsRepository.PushCliFunctionSource(LfsHashing);
|
||||
CliFunctionsRepository.PushCliFunctionSource(RemoteClient);
|
||||
const program = new Command();
|
||||
program.version('0.0.1');
|
||||
const properties = Object.getOwnPropertyNames(Input);
|
||||
core.info(`\n`);
|
||||
core.info(`INPUT:`);
|
||||
const actionYamlReader: ActionYamlReader = new ActionYamlReader();
|
||||
for (const element of properties) {
|
||||
program.option(`--${element} <${element}>`, actionYamlReader.GetActionYamlValue(element));
|
||||
if (Input[element] !== undefined && Input[element] !== '' && typeof Input[element] !== `function`) {
|
||||
}
|
||||
program.option(
|
||||
'-m, --mode <mode>',
|
||||
CliFunctionsRepository.GetAllCliModes()
|
||||
.map((x) => `${x.key} (${x.description})`)
|
||||
.join(` | `),
|
||||
);
|
||||
program.option('--populateOverride <populateOverride>', 'should use override query to pull input false by default');
|
||||
program.option('--cachePushFrom <cachePushFrom>', 'cache push from source folder');
|
||||
program.option('--cachePushTo <cachePushTo>', 'cache push to caching folder');
|
||||
program.option('--artifactName <artifactName>', 'caching artifact name');
|
||||
program.parse(process.argv);
|
||||
Cli.options = program.opts();
|
||||
return Cli.isCliMode;
|
||||
}
|
||||
|
||||
static async RunCli(): Promise<void> {
|
||||
Input.githubInputEnabled = false;
|
||||
if (Cli.options['populateOverride'] === `true`) {
|
||||
await CloudRunnerQueryOverride.PopulateQueryOverrideInput();
|
||||
}
|
||||
Cli.logInput();
|
||||
const results = CliFunctionsRepository.GetCliFunctions(Cli.options.mode);
|
||||
CloudRunnerLogger.log(`Entrypoint: ${results.key}`);
|
||||
Cli.options.versioning = 'None';
|
||||
return await results.target[results.propertyKey]();
|
||||
}
|
||||
|
||||
@CliFunction(`print-input`, `prints all input`)
|
||||
private static logInput() {
|
||||
core.info(`\n`);
|
||||
core.info(`INPUT:`);
|
||||
const properties = Object.getOwnPropertyNames(Input);
|
||||
for (const element of properties) {
|
||||
if (
|
||||
Input[element] !== undefined &&
|
||||
Input[element] !== '' &&
|
||||
typeof Input[element] !== `function` &&
|
||||
element !== 'length' &&
|
||||
element !== 'cliOptions' &&
|
||||
element !== 'prototype'
|
||||
) {
|
||||
core.info(`${element} ${Input[element]}`);
|
||||
}
|
||||
}
|
||||
core.info(`\n`);
|
||||
program.option(
|
||||
'-m, --mode <mode>',
|
||||
GetAllCliModes()
|
||||
.map((x) => `${x.key} (${x.description})`)
|
||||
.join(` | `),
|
||||
);
|
||||
program.parse(process.argv);
|
||||
|
||||
return program.opts();
|
||||
}
|
||||
|
||||
@CliFunction(`cli`, `runs a cloud runner build`)
|
||||
@ -60,29 +90,4 @@ export class CLI {
|
||||
const baseImage = new ImageTag(buildParameter);
|
||||
return await CloudRunner.run(buildParameter, baseImage.toString());
|
||||
}
|
||||
|
||||
@CliFunction(`remote-cli`, `sets up a repository, usually before a game-ci build`)
|
||||
static async runRemoteClientJob() {
|
||||
const buildParameter = JSON.parse(process.env.BUILD_PARAMETERS || '{}');
|
||||
RemoteClientLogger.log(`Build Params:
|
||||
${JSON.stringify(buildParameter, undefined, 4)}
|
||||
`);
|
||||
CloudRunnerState.setup(buildParameter);
|
||||
await SetupCloudRunnerRepository.run();
|
||||
}
|
||||
|
||||
@CliFunction(`cach-push`, `push to cache`)
|
||||
static async cachePush() {}
|
||||
|
||||
@CliFunction(`cach-pull`, `pull from cache`)
|
||||
static async cachePull() {}
|
||||
|
||||
@CliFunction(`garbage-collect-aws`, `garbage collect aws`)
|
||||
static async garbageCollectAws() {
|
||||
process.env.AWS_REGION = Input.region;
|
||||
const CF = new SDK.CloudFormation();
|
||||
|
||||
const stacks = await CF.listStacks().promise();
|
||||
CloudRunnerLogger.log(JSON.stringify(stacks, undefined, 4));
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,117 +0,0 @@
|
||||
import { assert } from 'console';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { Input } from '../../..';
|
||||
import CloudRunnerLogger from '../../../cloud-runner/services/cloud-runner-logger';
|
||||
import { CloudRunnerState } from '../../../cloud-runner/state/cloud-runner-state';
|
||||
import { CloudRunnerSystem } from './cloud-runner-system';
|
||||
import { LFSHashing } from './lfs-hashing';
|
||||
import { RemoteClientLogger } from './remote-client-logger';
|
||||
|
||||
export class Caching {
|
||||
public static async PushToCache(cacheFolder: string, sourceFolder: string, cacheKey: string) {
|
||||
const startPath = process.cwd();
|
||||
try {
|
||||
if (!fs.existsSync(cacheFolder)) {
|
||||
await CloudRunnerSystem.Run(`mkdir -p ${cacheFolder}`);
|
||||
}
|
||||
process.chdir(path.resolve(sourceFolder, '..'));
|
||||
|
||||
if (Input.cloudRunnerTests) {
|
||||
CloudRunnerLogger.log(
|
||||
`Hashed cache folder ${await LFSHashing.hashAllFiles(sourceFolder)} ${sourceFolder} ${path.basename(
|
||||
sourceFolder,
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (Input.cloudRunnerTests) {
|
||||
await CloudRunnerSystem.Run(`ls ${path.basename(sourceFolder)}`);
|
||||
}
|
||||
await CloudRunnerSystem.Run(`zip ${cacheKey}.zip ${path.basename(sourceFolder)}`);
|
||||
assert(fs.existsSync(`${cacheKey}.zip`), 'cache zip exists');
|
||||
assert(fs.existsSync(path.basename(sourceFolder)), 'source folder exists');
|
||||
await CloudRunnerSystem.Run(`mv ${cacheKey}.zip ${cacheFolder}`);
|
||||
RemoteClientLogger.log(`moved ${cacheKey}.zip to ${cacheFolder}`);
|
||||
assert(fs.existsSync(`${path.join(cacheFolder, cacheKey)}.zip`), 'cache zip exists inside cache folder');
|
||||
|
||||
if (Input.cloudRunnerTests) {
|
||||
await CloudRunnerSystem.Run(`ls ${cacheFolder}`);
|
||||
}
|
||||
} catch (error) {
|
||||
process.chdir(`${startPath}`);
|
||||
throw error;
|
||||
}
|
||||
process.chdir(`${startPath}`);
|
||||
}
|
||||
public static async PullFromCache(cacheFolder: string, destinationFolder: string, cacheKey: string = ``) {
|
||||
const startPath = process.cwd();
|
||||
RemoteClientLogger.log(`Caching for ${path.basename(destinationFolder)}`);
|
||||
try {
|
||||
if (!fs.existsSync(cacheFolder)) {
|
||||
fs.mkdirSync(cacheFolder);
|
||||
}
|
||||
|
||||
if (!fs.existsSync(destinationFolder)) {
|
||||
fs.mkdirSync(destinationFolder);
|
||||
}
|
||||
|
||||
const latestInBranch = await (await CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .zip$ | head -1`))
|
||||
.replace(/\n/g, ``)
|
||||
.replace('.zip', '');
|
||||
|
||||
process.chdir(cacheFolder);
|
||||
|
||||
const cacheSelection = cacheKey !== `` && fs.existsSync(`${cacheKey}.zip`) ? cacheKey : latestInBranch;
|
||||
await CloudRunnerLogger.log(`cache key ${cacheKey} selection ${cacheSelection}`);
|
||||
|
||||
if (fs.existsSync(`${cacheSelection}.zip`)) {
|
||||
const resultsFolder = `results${CloudRunnerState.buildParams.buildGuid}`;
|
||||
await CloudRunnerSystem.Run(`mkdir -p ${resultsFolder}`);
|
||||
if (Input.cloudRunnerTests) {
|
||||
await CloudRunnerSystem.Run(`tree ${destinationFolder}`);
|
||||
}
|
||||
RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.zip`);
|
||||
assert(`${fs.existsSync(destinationFolder)}`);
|
||||
assert(`${fs.existsSync(`${cacheSelection}.zip`)}`);
|
||||
const fullResultsFolder = path.join(cacheFolder, resultsFolder);
|
||||
if (Input.cloudRunnerTests) {
|
||||
await CloudRunnerSystem.Run(`tree ${cacheFolder}`);
|
||||
}
|
||||
await CloudRunnerSystem.Run(`unzip ${cacheSelection}.zip -d ${path.basename(resultsFolder)}`);
|
||||
RemoteClientLogger.log(`cache item extracted to ${fullResultsFolder}`);
|
||||
assert(`${fs.existsSync(fullResultsFolder)}`);
|
||||
const destinationParentFolder = path.resolve(destinationFolder, '..');
|
||||
if (fs.existsSync(destinationFolder)) {
|
||||
fs.rmSync(destinationFolder, { recursive: true, force: true });
|
||||
}
|
||||
await CloudRunnerSystem.Run(
|
||||
`mv "${fullResultsFolder}/${path.basename(destinationFolder)}" "${destinationParentFolder}"`,
|
||||
);
|
||||
if (Input.cloudRunnerTests) {
|
||||
await CloudRunnerSystem.Run(`tree ${destinationParentFolder}`);
|
||||
}
|
||||
} else {
|
||||
RemoteClientLogger.logWarning(`cache item ${cacheKey} doesn't exist ${destinationFolder}`);
|
||||
if (cacheSelection !== ``) {
|
||||
if (Input.cloudRunnerTests) {
|
||||
await CloudRunnerSystem.Run(`tree ${cacheFolder}`);
|
||||
}
|
||||
RemoteClientLogger.logWarning(`cache item ${cacheKey}.zip doesn't exist ${destinationFolder}`);
|
||||
throw new Error(`Failed to get cache item, but cache hit was found: ${cacheSelection}`);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
process.chdir(`${startPath}`);
|
||||
throw error;
|
||||
}
|
||||
process.chdir(`${startPath}`);
|
||||
}
|
||||
|
||||
public static handleCachePurging() {
|
||||
if (process.env.PURGE_REMOTE_BUILDER_CACHE !== undefined) {
|
||||
RemoteClientLogger.log(`purging ${CloudRunnerState.purgeRemoteCaching}`);
|
||||
fs.rmdirSync(CloudRunnerState.cacheFolder, { recursive: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,37 +0,0 @@
|
||||
import { exec } from 'child_process';
|
||||
import { RemoteClientLogger } from './remote-client-logger';
|
||||
|
||||
export class CloudRunnerSystem {
|
||||
public static async Run(command: string, suppressError = false) {
|
||||
for (const element of command.split(`\n`)) {
|
||||
RemoteClientLogger.log(element);
|
||||
}
|
||||
return await new Promise<string>((promise) => {
|
||||
let output = '';
|
||||
const child = exec(command, (error, stdout, stderr) => {
|
||||
if (error && !suppressError) {
|
||||
throw error;
|
||||
}
|
||||
if (stderr) {
|
||||
const diagnosticOutput = `${stderr.toString()}`;
|
||||
RemoteClientLogger.logCliDiagnostic(diagnosticOutput);
|
||||
output += diagnosticOutput;
|
||||
return;
|
||||
}
|
||||
const outputChunk = `${stdout}`;
|
||||
output += outputChunk;
|
||||
});
|
||||
child.on('close', function (code) {
|
||||
RemoteClientLogger.log(`[Exit code ${code}]`);
|
||||
if (code !== 0 && !suppressError) {
|
||||
throw new Error(output);
|
||||
}
|
||||
const outputLines = output.split(`\n`);
|
||||
for (const element of outputLines) {
|
||||
RemoteClientLogger.log(element);
|
||||
}
|
||||
promise(output);
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
@ -1,81 +0,0 @@
|
||||
import fs from 'fs';
|
||||
import { CloudRunnerState } from '../../cloud-runner/state/cloud-runner-state';
|
||||
import { Caching } from './remote-client-services/caching';
|
||||
import { LFSHashing } from './remote-client-services/lfs-hashing';
|
||||
import { CloudRunnerSystem } from './remote-client-services/cloud-runner-system';
|
||||
import { Input } from '../..';
|
||||
import { RemoteClientLogger } from './remote-client-services/remote-client-logger';
|
||||
import path from 'path';
|
||||
import { assert } from 'console';
|
||||
|
||||
export class SetupCloudRunnerRepository {
|
||||
public static async run() {
|
||||
try {
|
||||
await CloudRunnerSystem.Run(`mkdir -p ${CloudRunnerState.buildPathFull}`);
|
||||
await CloudRunnerSystem.Run(`mkdir -p ${CloudRunnerState.repoPathFull}`);
|
||||
await CloudRunnerSystem.Run(`mkdir -p ${CloudRunnerState.cacheFolderFull}`);
|
||||
|
||||
process.chdir(CloudRunnerState.repoPathFull);
|
||||
if (Input.cloudRunnerTests) {
|
||||
await CloudRunnerSystem.Run(`ls -lh`);
|
||||
await CloudRunnerSystem.Run(`tree`);
|
||||
}
|
||||
await SetupCloudRunnerRepository.cloneRepoWithoutLFSFiles();
|
||||
if (Input.cloudRunnerTests) {
|
||||
await CloudRunnerSystem.Run(`ls -lh`);
|
||||
await CloudRunnerSystem.Run(`tree`);
|
||||
}
|
||||
const lfsHashes = await LFSHashing.createLFSHashFiles();
|
||||
if (fs.existsSync(CloudRunnerState.libraryFolderFull)) {
|
||||
RemoteClientLogger.logWarning(`!Warning!: The Unity library was included in the git repository`);
|
||||
}
|
||||
await Caching.PullFromCache(
|
||||
CloudRunnerState.lfsCacheFolderFull,
|
||||
CloudRunnerState.lfsDirectoryFull,
|
||||
`${lfsHashes.lfsGuid}`,
|
||||
);
|
||||
await SetupCloudRunnerRepository.pullLatestLFS();
|
||||
await Caching.PushToCache(
|
||||
CloudRunnerState.lfsCacheFolderFull,
|
||||
CloudRunnerState.lfsDirectoryFull,
|
||||
`${lfsHashes.lfsGuid}`,
|
||||
);
|
||||
await Caching.PullFromCache(CloudRunnerState.libraryCacheFolderFull, CloudRunnerState.libraryFolderFull);
|
||||
Caching.handleCachePurging();
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private static async cloneRepoWithoutLFSFiles() {
|
||||
try {
|
||||
process.chdir(`${CloudRunnerState.repoPathFull}`);
|
||||
RemoteClientLogger.log(`Initializing source repository for cloning with caching of LFS files`);
|
||||
await CloudRunnerSystem.Run(`git config --global advice.detachedHead false`);
|
||||
RemoteClientLogger.log(`Cloning the repository being built:`);
|
||||
await CloudRunnerSystem.Run(`git lfs install --skip-smudge`);
|
||||
await CloudRunnerSystem.Run(
|
||||
`git clone -b ${CloudRunnerState.branchName} ${CloudRunnerState.targetBuildRepoUrl} ${path.resolve(
|
||||
`..`,
|
||||
path.basename(CloudRunnerState.repoPathFull),
|
||||
)}`,
|
||||
);
|
||||
assert(fs.existsSync(`.git`));
|
||||
RemoteClientLogger.log(`${CloudRunnerState.buildParams.branch}`);
|
||||
await CloudRunnerSystem.Run(`git checkout ${CloudRunnerState.buildParams.branch}`);
|
||||
assert(fs.existsSync(path.join(`.git`, `lfs`)), 'LFS folder should not exist before caching');
|
||||
RemoteClientLogger.log(`Checked out ${process.env.GITHUB_SHA}`);
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private static async pullLatestLFS() {
|
||||
await CloudRunnerSystem.Run(`ls -lh ${CloudRunnerState.lfsDirectoryFull}/..`);
|
||||
process.chdir(CloudRunnerState.repoPathFull);
|
||||
await CloudRunnerSystem.Run(`git lfs pull`);
|
||||
RemoteClientLogger.log(`pulled latest LFS files`);
|
||||
assert(fs.existsSync(CloudRunnerState.lfsDirectoryFull));
|
||||
await CloudRunnerSystem.Run(`ls -lh ${CloudRunnerState.lfsDirectoryFull}/..`);
|
||||
}
|
||||
}
|
||||
@ -1,5 +1,5 @@
|
||||
import CloudRunnerEnvironmentVariable from '../services/cloud-runner-environment-variable';
|
||||
import CloudRunnerSecret from '../services/cloud-runner-secret';
|
||||
import CloudRunnerEnvironmentVariable from './services/cloud-runner-environment-variable';
|
||||
import CloudRunnerSecret from './services/cloud-runner-secret';
|
||||
|
||||
export class CloudRunnerStepState {
|
||||
public image: string;
|
||||
@ -4,6 +4,9 @@ import Input from '../input';
|
||||
import { CloudRunnerStatics } from './cloud-runner-statics';
|
||||
import { TaskParameterSerializer } from './services/task-parameter-serializer';
|
||||
import UnityVersioning from '../unity-versioning';
|
||||
import { Cli } from '../cli/cli';
|
||||
import CloudRunnerLogger from './services/cloud-runner-logger';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
|
||||
describe('Cloud Runner', () => {
|
||||
it('responds', () => {});
|
||||
@ -13,10 +16,12 @@ describe('Cloud Runner', () => {
|
||||
const testSecretValue = 'testSecretValue';
|
||||
if (Input.cloudRunnerTests) {
|
||||
it('All build parameters sent to cloud runner as env vars', async () => {
|
||||
Input.cliOptions = {
|
||||
// build parameters
|
||||
Cli.options = {
|
||||
versioning: 'None',
|
||||
projectPath: 'test-project',
|
||||
unityVersion: UnityVersioning.read('test-project'),
|
||||
targetPlatform: 'StandaloneLinux64',
|
||||
customJob: `
|
||||
- name: 'step 1'
|
||||
image: 'alpine'
|
||||
@ -27,9 +32,13 @@ describe('Cloud Runner', () => {
|
||||
`,
|
||||
};
|
||||
Input.githubInputEnabled = false;
|
||||
// setup parameters
|
||||
const buildParameter = await BuildParameters.create();
|
||||
Input.githubInputEnabled = true;
|
||||
const baseImage = new ImageTag(buildParameter);
|
||||
// run the job
|
||||
const file = await CloudRunner.run(buildParameter, baseImage.toString());
|
||||
// assert results
|
||||
expect(file).toContain(JSON.stringify(buildParameter));
|
||||
expect(file).toContain(`${Input.ToEnvVarFormat(testSecretName)}=${testSecretValue}`);
|
||||
const environmentVariables = TaskParameterSerializer.readBuildEnvironmentVariables();
|
||||
@ -41,10 +50,86 @@ describe('Cloud Runner', () => {
|
||||
if (typeof element.value === `string`) {
|
||||
element.value = element.value.replace(/\s+/g, '');
|
||||
}
|
||||
CloudRunnerLogger.log(`checking input/build param ${element.name} ${element.value}`);
|
||||
}
|
||||
}
|
||||
for (const element of environmentVariables) {
|
||||
if (element.value !== undefined && typeof element.value !== 'function') {
|
||||
expect(newLinePurgedFile).toContain(`${element.name}`);
|
||||
expect(newLinePurgedFile).toContain(`${element.name}=${element.value}`);
|
||||
}
|
||||
}
|
||||
delete Cli.options;
|
||||
}, 1000000);
|
||||
it('Run one build it should not use cache, run subsequent build which should use cache', async () => {
|
||||
Cli.options = {
|
||||
versioning: 'None',
|
||||
projectPath: 'test-project',
|
||||
unityVersion: UnityVersioning.determineUnityVersion('test-project', UnityVersioning.read('test-project')),
|
||||
targetPlatform: 'StandaloneLinux64',
|
||||
cacheKey: `test-case-${uuidv4()}`,
|
||||
};
|
||||
Input.githubInputEnabled = false;
|
||||
const buildParameter = await BuildParameters.create();
|
||||
const baseImage = new ImageTag(buildParameter);
|
||||
const results = await CloudRunner.run(buildParameter, baseImage.toString());
|
||||
const libraryString = 'Rebuilding Library because the asset database could not be found!';
|
||||
const buildSucceededString = 'Build succeeded';
|
||||
expect(results).toContain(libraryString);
|
||||
expect(results).toContain(buildSucceededString);
|
||||
CloudRunnerLogger.log(`run 1 succeeded`);
|
||||
const buildParameter2 = await BuildParameters.create();
|
||||
const baseImage2 = new ImageTag(buildParameter2);
|
||||
const results2 = await CloudRunner.run(buildParameter2, baseImage2.toString());
|
||||
CloudRunnerLogger.log(`run 2 succeeded`);
|
||||
expect(results2).toContain(buildSucceededString);
|
||||
expect(results2).toEqual(expect.not.stringContaining(libraryString));
|
||||
Input.githubInputEnabled = true;
|
||||
delete Cli.options;
|
||||
}, 1000000);
|
||||
}
|
||||
it('Local cloud runner returns commands', async () => {
|
||||
// build parameters
|
||||
Cli.options = {
|
||||
versioning: 'None',
|
||||
projectPath: 'test-project',
|
||||
unityVersion: UnityVersioning.read('test-project'),
|
||||
cloudRunnerCluster: 'local-system',
|
||||
targetPlatform: 'StandaloneLinux64',
|
||||
customJob: `
|
||||
- name: 'step 1'
|
||||
image: 'alpine'
|
||||
commands: 'dir'
|
||||
secrets:
|
||||
- name: '${testSecretName}'
|
||||
value: '${testSecretValue}'
|
||||
`,
|
||||
};
|
||||
Input.githubInputEnabled = false;
|
||||
// setup parameters
|
||||
const buildParameter = await BuildParameters.create();
|
||||
const baseImage = new ImageTag(buildParameter);
|
||||
// run the job
|
||||
await expect(CloudRunner.run(buildParameter, baseImage.toString())).resolves.not.toThrow();
|
||||
Input.githubInputEnabled = true;
|
||||
delete Cli.options;
|
||||
}, 1000000);
|
||||
it('Test cloud runner returns commands', async () => {
|
||||
// build parameters
|
||||
Cli.options = {
|
||||
versioning: 'None',
|
||||
projectPath: 'test-project',
|
||||
unityVersion: UnityVersioning.read('test-project'),
|
||||
cloudRunnerCluster: 'test',
|
||||
targetPlatform: 'StandaloneLinux64',
|
||||
};
|
||||
Input.githubInputEnabled = false;
|
||||
// setup parameters
|
||||
const buildParameter = await BuildParameters.create();
|
||||
const baseImage = new ImageTag(buildParameter);
|
||||
// run the job
|
||||
await expect(CloudRunner.run(buildParameter, baseImage.toString())).resolves.not.toThrow();
|
||||
Input.githubInputEnabled = true;
|
||||
delete Cli.options;
|
||||
}, 1000000);
|
||||
});
|
||||
|
||||
@ -1,35 +1,58 @@
|
||||
import AWSBuildPlatform from './aws';
|
||||
import { BuildParameters } from '..';
|
||||
import { CloudRunnerState } from './state/cloud-runner-state';
|
||||
import Kubernetes from './k8s';
|
||||
import AwsBuildPlatform from './providers/aws';
|
||||
import { BuildParameters, Input } from '..';
|
||||
import Kubernetes from './providers/k8s';
|
||||
import CloudRunnerLogger from './services/cloud-runner-logger';
|
||||
import { CloudRunnerStepState } from './state/cloud-runner-step-state';
|
||||
import { CloudRunnerStepState } from './cloud-runner-step-state';
|
||||
import { WorkflowCompositionRoot } from './workflows/workflow-composition-root';
|
||||
import { CloudRunnerError } from './error/cloud-runner-error';
|
||||
import { TaskParameterSerializer } from './services/task-parameter-serializer';
|
||||
import * as core from '@actions/core';
|
||||
import CloudRunnerSecret from './services/cloud-runner-secret';
|
||||
import { ProviderInterface } from './providers/provider-interface';
|
||||
import CloudRunnerEnvironmentVariable from './services/cloud-runner-environment-variable';
|
||||
import TestCloudRunner from './providers/test';
|
||||
import LocalCloudRunner from './providers/local';
|
||||
import LocalDockerCloudRunner from './providers/local-docker';
|
||||
|
||||
class CloudRunner {
|
||||
public static Provider: ProviderInterface;
|
||||
static buildParameters: BuildParameters;
|
||||
public static defaultSecrets: CloudRunnerSecret[];
|
||||
public static cloudRunnerEnvironmentVariables: CloudRunnerEnvironmentVariable[];
|
||||
private static setup(buildParameters: BuildParameters) {
|
||||
CloudRunnerLogger.setup();
|
||||
CloudRunnerState.setup(buildParameters);
|
||||
CloudRunner.buildParameters = buildParameters;
|
||||
CloudRunner.setupBuildPlatform();
|
||||
const parameters = TaskParameterSerializer.readBuildEnvironmentVariables();
|
||||
for (const element of parameters) {
|
||||
core.setOutput(element.name, element.value);
|
||||
CloudRunner.defaultSecrets = TaskParameterSerializer.readDefaultSecrets();
|
||||
CloudRunner.cloudRunnerEnvironmentVariables = TaskParameterSerializer.readBuildEnvironmentVariables();
|
||||
if (!buildParameters.isCliMode) {
|
||||
const buildParameterPropertyNames = Object.getOwnPropertyNames(buildParameters);
|
||||
for (const element of CloudRunner.cloudRunnerEnvironmentVariables) {
|
||||
core.setOutput(Input.ToEnvVarFormat(element.name), element.value);
|
||||
}
|
||||
for (const element of buildParameterPropertyNames) {
|
||||
core.setOutput(Input.ToEnvVarFormat(element), buildParameters[element]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static setupBuildPlatform() {
|
||||
switch (CloudRunnerState.buildParams.cloudRunnerCluster) {
|
||||
CloudRunnerLogger.log(`Cloud Runner platform selected ${CloudRunner.buildParameters.cloudRunnerCluster}`);
|
||||
switch (CloudRunner.buildParameters.cloudRunnerCluster) {
|
||||
case 'k8s':
|
||||
CloudRunnerLogger.log('Cloud Runner platform selected Kubernetes');
|
||||
CloudRunnerState.CloudRunnerProviderPlatform = new Kubernetes(CloudRunnerState.buildParams);
|
||||
CloudRunner.Provider = new Kubernetes(CloudRunner.buildParameters);
|
||||
break;
|
||||
default:
|
||||
case 'aws':
|
||||
CloudRunnerLogger.log('Cloud Runner platform selected AWS');
|
||||
CloudRunnerState.CloudRunnerProviderPlatform = new AWSBuildPlatform(CloudRunnerState.buildParams);
|
||||
CloudRunner.Provider = new AwsBuildPlatform(CloudRunner.buildParameters);
|
||||
break;
|
||||
case 'test':
|
||||
CloudRunner.Provider = new TestCloudRunner();
|
||||
break;
|
||||
case 'local-system':
|
||||
CloudRunner.Provider = new LocalCloudRunner();
|
||||
break;
|
||||
case 'local-docker':
|
||||
CloudRunner.Provider = new LocalDockerCloudRunner();
|
||||
break;
|
||||
}
|
||||
}
|
||||
@ -37,33 +60,29 @@ class CloudRunner {
|
||||
static async run(buildParameters: BuildParameters, baseImage: string) {
|
||||
CloudRunner.setup(buildParameters);
|
||||
try {
|
||||
core.startGroup('Setup remote runner');
|
||||
await CloudRunnerState.CloudRunnerProviderPlatform.setupSharedResources(
|
||||
CloudRunnerState.buildParams.buildGuid,
|
||||
CloudRunnerState.buildParams,
|
||||
CloudRunnerState.branchName,
|
||||
CloudRunnerState.defaultSecrets,
|
||||
if (!CloudRunner.buildParameters.isCliMode) core.startGroup('Setup shared cloud runner resources');
|
||||
await CloudRunner.Provider.setup(
|
||||
CloudRunner.buildParameters.buildGuid,
|
||||
CloudRunner.buildParameters,
|
||||
CloudRunner.buildParameters.branch,
|
||||
CloudRunner.defaultSecrets,
|
||||
);
|
||||
core.endGroup();
|
||||
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
|
||||
const output = await new WorkflowCompositionRoot().run(
|
||||
new CloudRunnerStepState(
|
||||
baseImage,
|
||||
TaskParameterSerializer.readBuildEnvironmentVariables(),
|
||||
CloudRunnerState.defaultSecrets,
|
||||
),
|
||||
new CloudRunnerStepState(baseImage, CloudRunner.cloudRunnerEnvironmentVariables, CloudRunner.defaultSecrets),
|
||||
);
|
||||
core.startGroup('Cleanup');
|
||||
await CloudRunnerState.CloudRunnerProviderPlatform.cleanupSharedResources(
|
||||
CloudRunnerState.buildParams.buildGuid,
|
||||
CloudRunnerState.buildParams,
|
||||
CloudRunnerState.branchName,
|
||||
CloudRunnerState.defaultSecrets,
|
||||
if (!CloudRunner.buildParameters.isCliMode) core.startGroup('Cleanup shared cloud runner resources');
|
||||
await CloudRunner.Provider.cleanup(
|
||||
CloudRunner.buildParameters.buildGuid,
|
||||
CloudRunner.buildParameters,
|
||||
CloudRunner.buildParameters.branch,
|
||||
CloudRunner.defaultSecrets,
|
||||
);
|
||||
CloudRunnerLogger.log(`Cleanup complete`);
|
||||
core.endGroup();
|
||||
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
|
||||
return output;
|
||||
} catch (error) {
|
||||
core.endGroup();
|
||||
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
|
||||
await CloudRunnerError.handleException(error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
@ -1,16 +1,16 @@
|
||||
import CloudRunnerLogger from '../services/cloud-runner-logger';
|
||||
import { CloudRunnerState } from '../state/cloud-runner-state';
|
||||
import * as core from '@actions/core';
|
||||
import CloudRunner from '../cloud-runner';
|
||||
|
||||
export class CloudRunnerError {
|
||||
public static async handleException(error: unknown) {
|
||||
CloudRunnerLogger.error(JSON.stringify(error, undefined, 4));
|
||||
core.setFailed('Cloud Runner failed');
|
||||
await CloudRunnerState.CloudRunnerProviderPlatform.cleanupSharedResources(
|
||||
CloudRunnerState.buildParams.buildGuid,
|
||||
CloudRunnerState.buildParams,
|
||||
CloudRunnerState.branchName,
|
||||
CloudRunnerState.defaultSecrets,
|
||||
await CloudRunner.Provider.cleanup(
|
||||
CloudRunner.buildParameters.buildGuid,
|
||||
CloudRunner.buildParameters,
|
||||
CloudRunner.buildParameters.branch,
|
||||
CloudRunner.defaultSecrets,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import CloudRunnerLogger from '../services/cloud-runner-logger';
|
||||
import CloudRunnerLogger from '../../services/cloud-runner-logger';
|
||||
import * as core from '@actions/core';
|
||||
import * as SDK from 'aws-sdk';
|
||||
import * as fs from 'fs';
|
||||
@ -1,6 +1,6 @@
|
||||
import * as fs from 'fs';
|
||||
|
||||
export class AWSTemplates {
|
||||
export class AWSCloudFormationTemplates {
|
||||
public static getParameterTemplate(p1) {
|
||||
return `
|
||||
${p1}:
|
||||
@ -1,13 +1,13 @@
|
||||
import CloudRunnerLogger from '../services/cloud-runner-logger';
|
||||
import CloudRunnerLogger from '../../services/cloud-runner-logger';
|
||||
import * as SDK from 'aws-sdk';
|
||||
import * as core from '@actions/core';
|
||||
import { Input } from '../..';
|
||||
import CloudRunner from '../../cloud-runner';
|
||||
|
||||
export class AWSError {
|
||||
static async handleStackCreationFailure(error: any, CF: SDK.CloudFormation, taskDefStackName: string) {
|
||||
CloudRunnerLogger.log('aws error: ');
|
||||
core.error(JSON.stringify(error, undefined, 4));
|
||||
if (Input.cloudRunnerTests) {
|
||||
if (CloudRunner.buildParameters.cloudRunnerIntegrationTests) {
|
||||
CloudRunnerLogger.log('Getting events and resources for task stack');
|
||||
const events = (await CF.describeStackEvents({ StackName: taskDefStackName }).promise()).StackEvents;
|
||||
CloudRunnerLogger.log(JSON.stringify(events, undefined, 4));
|
||||
@ -1,8 +1,8 @@
|
||||
import * as SDK from 'aws-sdk';
|
||||
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
|
||||
import CloudRunnerSecret from '../services/cloud-runner-secret';
|
||||
import { AWSTemplates } from './aws-templates';
|
||||
import CloudRunnerLogger from '../services/cloud-runner-logger';
|
||||
import CloudRunnerSecret from '../../services/cloud-runner-secret';
|
||||
import { AWSCloudFormationTemplates } from './aws-cloud-formation-templates';
|
||||
import CloudRunnerLogger from '../../services/cloud-runner-logger';
|
||||
import { AWSError } from './aws-error';
|
||||
|
||||
export class AWSJobStack {
|
||||
@ -22,7 +22,7 @@ export class AWSJobStack {
|
||||
secrets: CloudRunnerSecret[],
|
||||
): Promise<CloudRunnerAWSTaskDef> {
|
||||
const taskDefStackName = `${this.baseStackName}-${buildGuid}`;
|
||||
let taskDefCloudFormation = AWSTemplates.readTaskCloudFormationTemplate();
|
||||
let taskDefCloudFormation = AWSCloudFormationTemplates.readTaskCloudFormationTemplate();
|
||||
for (const secret of secrets) {
|
||||
secret.ParameterKey = `${buildGuid.replace(/[^\dA-Za-z]/g, '')}${secret.ParameterKey.replace(
|
||||
/[^\dA-Za-z]/g,
|
||||
@ -35,20 +35,20 @@ export class AWSJobStack {
|
||||
secrets = secrets.filter((x) => x !== secret);
|
||||
continue;
|
||||
}
|
||||
taskDefCloudFormation = AWSTemplates.insertAtTemplate(
|
||||
taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
|
||||
taskDefCloudFormation,
|
||||
'p1 - input',
|
||||
AWSTemplates.getParameterTemplate(secret.ParameterKey),
|
||||
AWSCloudFormationTemplates.getParameterTemplate(secret.ParameterKey),
|
||||
);
|
||||
taskDefCloudFormation = AWSTemplates.insertAtTemplate(
|
||||
taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
|
||||
taskDefCloudFormation,
|
||||
'p2 - secret',
|
||||
AWSTemplates.getSecretTemplate(`${secret.ParameterKey}`),
|
||||
AWSCloudFormationTemplates.getSecretTemplate(`${secret.ParameterKey}`),
|
||||
);
|
||||
taskDefCloudFormation = AWSTemplates.insertAtTemplate(
|
||||
taskDefCloudFormation = AWSCloudFormationTemplates.insertAtTemplate(
|
||||
taskDefCloudFormation,
|
||||
'p3 - container def',
|
||||
AWSTemplates.getSecretDefinitionTemplate(secret.EnvironmentVariable, secret.ParameterKey),
|
||||
AWSCloudFormationTemplates.getSecretDefinitionTemplate(secret.EnvironmentVariable, secret.ParameterKey),
|
||||
);
|
||||
}
|
||||
const secretsMappedToCloudFormationParameters = secrets.map((x) => {
|
||||
@ -1,13 +1,13 @@
|
||||
import * as AWS from 'aws-sdk';
|
||||
import CloudRunnerEnvironmentVariable from '../services/cloud-runner-environment-variable';
|
||||
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
|
||||
import * as core from '@actions/core';
|
||||
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
|
||||
import * as zlib from 'zlib';
|
||||
import CloudRunnerLogger from '../services/cloud-runner-logger';
|
||||
import { Input } from '../..';
|
||||
import { CloudRunnerState } from '../state/cloud-runner-state';
|
||||
import { CloudRunnerStatics } from '../cloud-runner-statics';
|
||||
import { CloudRunnerBuildCommandProcessor } from '../services/cloud-runner-build-command-process';
|
||||
import CloudRunnerLogger from '../../services/cloud-runner-logger';
|
||||
import { Input } from '../../..';
|
||||
import CloudRunner from '../../cloud-runner';
|
||||
import { CloudRunnerStatics } from '../../cloud-runner-statics';
|
||||
import { CloudRunnerBuildCommandProcessor } from '../../services/cloud-runner-build-command-process';
|
||||
|
||||
class AWSTaskRunner {
|
||||
static async runTask(
|
||||
@ -39,7 +39,7 @@ class AWSTaskRunner {
|
||||
{
|
||||
name: taskDef.taskDefStackName,
|
||||
environment,
|
||||
command: ['-c', CloudRunnerBuildCommandProcessor.ProcessCommands(commands, CloudRunnerState.buildParams)],
|
||||
command: ['-c', CloudRunnerBuildCommandProcessor.ProcessCommands(commands, CloudRunner.buildParameters)],
|
||||
},
|
||||
],
|
||||
},
|
||||
@ -52,10 +52,32 @@ class AWSTaskRunner {
|
||||
},
|
||||
},
|
||||
}).promise();
|
||||
|
||||
CloudRunnerLogger.log('Cloud runner job is starting');
|
||||
const taskArn = task.tasks?.[0].taskArn || '';
|
||||
CloudRunnerLogger.log('Cloud runner job is starting');
|
||||
await AWSTaskRunner.waitUntilTaskRunning(ECS, taskArn, cluster);
|
||||
CloudRunnerLogger.log(
|
||||
`Cloud runner job status is running ${(await AWSTaskRunner.describeTasks(ECS, cluster, taskArn))?.lastStatus}`,
|
||||
);
|
||||
const output = await this.streamLogsUntilTaskStops(ECS, CF, taskDef, cluster, taskArn, streamName);
|
||||
const taskData = await AWSTaskRunner.describeTasks(ECS, cluster, taskArn);
|
||||
const exitCode = taskData.containers?.[0].exitCode;
|
||||
const wasSuccessful = exitCode === 0 || (exitCode === undefined && taskData.lastStatus === 'RUNNING');
|
||||
if (wasSuccessful) {
|
||||
CloudRunnerLogger.log(`Cloud runner job has finished successfully`);
|
||||
return output;
|
||||
} else {
|
||||
if (taskData.stoppedReason === 'Essential container in task exited' && exitCode === 1) {
|
||||
throw new Error('Container exited with code 1');
|
||||
}
|
||||
const message = `Cloud runner job exit code ${exitCode}`;
|
||||
taskData.overrides = undefined;
|
||||
taskData.attachments = undefined;
|
||||
CloudRunnerLogger.log(`${message} ${JSON.stringify(taskData, undefined, 4)}`);
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
private static async waitUntilTaskRunning(ECS: AWS.ECS, taskArn: string, cluster: string) {
|
||||
try {
|
||||
await ECS.waitFor('tasksRunning', { tasks: [taskArn], cluster }).promise();
|
||||
} catch (error_) {
|
||||
@ -70,24 +92,6 @@ class AWSTaskRunner {
|
||||
core.setFailed(error);
|
||||
core.error(error);
|
||||
}
|
||||
CloudRunnerLogger.log(`Cloud runner job is running`);
|
||||
|
||||
const output = await this.streamLogsUntilTaskStops(ECS, CF, taskDef, cluster, taskArn, streamName);
|
||||
const exitCode = (await AWSTaskRunner.describeTasks(ECS, cluster, taskArn)).containers?.[0].exitCode;
|
||||
CloudRunnerLogger.log(`Cloud runner job exit code ${exitCode}`);
|
||||
if (exitCode !== 0 && exitCode !== undefined) {
|
||||
core.error(
|
||||
`job failed with exit code ${exitCode} ${JSON.stringify(
|
||||
await ECS.describeTasks({ tasks: [taskArn], cluster }).promise(),
|
||||
undefined,
|
||||
4,
|
||||
)}`,
|
||||
);
|
||||
throw new Error(`job failed with exit code ${exitCode}`);
|
||||
} else {
|
||||
CloudRunnerLogger.log(`Cloud runner job has finished successfully`);
|
||||
return output;
|
||||
}
|
||||
}
|
||||
|
||||
static async describeTasks(ECS: AWS.ECS, clusterName: string, taskArn: string) {
|
||||
@ -114,10 +118,6 @@ class AWSTaskRunner {
|
||||
const stream = await AWSTaskRunner.getLogStream(kinesis, kinesisStreamName);
|
||||
let iterator = await AWSTaskRunner.getLogIterator(kinesis, stream);
|
||||
|
||||
CloudRunnerLogger.log(
|
||||
`Cloud runner job status is ${(await AWSTaskRunner.describeTasks(ECS, clusterName, taskArn))?.lastStatus}`,
|
||||
);
|
||||
|
||||
const logBaseUrl = `https://${Input.region}.console.aws.amazon.com/cloudwatch/home?region=${CF.config.region}#logsV2:log-groups/log-group/${taskDef.taskDefStackName}`;
|
||||
CloudRunnerLogger.log(`You can also see the logs at AWS Cloud Watch: ${logBaseUrl}`);
|
||||
let shouldReadLogs = true;
|
||||
@ -156,6 +156,9 @@ class AWSTaskRunner {
|
||||
}
|
||||
|
||||
private static checkStreamingShouldContinue(taskData: AWS.ECS.Task, timestamp: number, shouldReadLogs: boolean) {
|
||||
if (taskData?.lastStatus === 'UNKNOWN') {
|
||||
CloudRunnerLogger.log('## Cloud runner job unknwon');
|
||||
}
|
||||
if (taskData?.lastStatus !== 'RUNNING') {
|
||||
if (timestamp === 0) {
|
||||
CloudRunnerLogger.log('## Cloud runner job stopped, streaming end of logs');
|
||||
@ -185,14 +188,19 @@ class AWSTaskRunner {
|
||||
if (json.messageType === 'DATA_MESSAGE') {
|
||||
for (let logEventsIndex = 0; logEventsIndex < json.logEvents.length; logEventsIndex++) {
|
||||
let message = json.logEvents[logEventsIndex].message;
|
||||
if (json.logEvents[logEventsIndex].message.includes(`---${CloudRunnerState.buildParams.logId}`)) {
|
||||
if (json.logEvents[logEventsIndex].message.includes(`---${CloudRunner.buildParameters.logId}`)) {
|
||||
CloudRunnerLogger.log('End of log transmission received');
|
||||
shouldReadLogs = false;
|
||||
} else if (message.includes('Rebuilding Library because the asset database could not be found!')) {
|
||||
core.warning('LIBRARY NOT FOUND!');
|
||||
} else if (message.includes('Build succeeded')) {
|
||||
core.setOutput('build-result', 'success');
|
||||
} else if (message.includes('Build fail')) {
|
||||
core.setOutput('build-result', 'failed');
|
||||
core.error('BUILD FAILED!');
|
||||
}
|
||||
message = `[${CloudRunnerStatics.logPrefix}] ${message}`;
|
||||
if (Input.cloudRunnerTests) {
|
||||
if (CloudRunner.buildParameters.cloudRunnerIntegrationTests) {
|
||||
output += message;
|
||||
}
|
||||
CloudRunnerLogger.log(message);
|
||||
@ -0,0 +1,30 @@
|
||||
import AWS from 'aws-sdk';
|
||||
import { CliFunction } from '../../../../cli/cli-functions-repository';
|
||||
import Input from '../../../../input';
|
||||
import CloudRunnerLogger from '../../../services/cloud-runner-logger';
|
||||
|
||||
export class AwsCliCommands {
|
||||
@CliFunction(`aws-garbage-collect`, `garbage collect aws`)
|
||||
static async garbageCollectAws() {
|
||||
process.env.AWS_REGION = Input.region;
|
||||
CloudRunnerLogger.log(`Cloud Formation stacks`);
|
||||
const CF = new AWS.CloudFormation();
|
||||
const stacks =
|
||||
(await CF.listStacks().promise()).StackSummaries?.filter((_x) => _x.StackStatus !== 'DELETE_COMPLETE') || [];
|
||||
for (const element of stacks) {
|
||||
CloudRunnerLogger.log(JSON.stringify(element, undefined, 4));
|
||||
}
|
||||
CloudRunnerLogger.log(`ECS Clusters`);
|
||||
const ecs = new AWS.ECS();
|
||||
const clusters = (await ecs.listClusters().promise()).clusterArns || [];
|
||||
if (stacks === undefined) {
|
||||
return;
|
||||
}
|
||||
for (const element of clusters) {
|
||||
const input: AWS.ECS.ListTasksRequest = {
|
||||
cluster: element,
|
||||
};
|
||||
CloudRunnerLogger.log(JSON.stringify(await ecs.listTasks(input).promise(), undefined, 4));
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,22 +1,22 @@
|
||||
import * as SDK from 'aws-sdk';
|
||||
import CloudRunnerSecret from '../services/cloud-runner-secret';
|
||||
import CloudRunnerEnvironmentVariable from '../services/cloud-runner-environment-variable';
|
||||
import CloudRunnerSecret from '../../services/cloud-runner-secret';
|
||||
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
|
||||
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
|
||||
import AWSTaskRunner from './aws-task-runner';
|
||||
import { CloudRunnerProviderInterface } from '../services/cloud-runner-provider-interface';
|
||||
import BuildParameters from '../../build-parameters';
|
||||
import CloudRunnerLogger from '../services/cloud-runner-logger';
|
||||
import { ProviderInterface } from '../provider-interface';
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import CloudRunnerLogger from '../../services/cloud-runner-logger';
|
||||
import { AWSJobStack } from './aws-job-stack';
|
||||
import { AWSBaseStack } from './aws-base-stack';
|
||||
import { Input } from '../..';
|
||||
import { Input } from '../../..';
|
||||
|
||||
class AWSBuildEnvironment implements CloudRunnerProviderInterface {
|
||||
class AWSBuildEnvironment implements ProviderInterface {
|
||||
private baseStackName: string;
|
||||
|
||||
constructor(buildParameters: BuildParameters) {
|
||||
this.baseStackName = buildParameters.awsBaseStackName;
|
||||
}
|
||||
async cleanupSharedResources(
|
||||
async cleanup(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
@ -26,7 +26,7 @@ class AWSBuildEnvironment implements CloudRunnerProviderInterface {
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {}
|
||||
async setupSharedResources(
|
||||
async setup(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
@ -1,19 +1,20 @@
|
||||
import * as k8s from '@kubernetes/client-node';
|
||||
import { BuildParameters, Output } from '../..';
|
||||
import { BuildParameters, Output } from '../../..';
|
||||
import * as core from '@actions/core';
|
||||
import { CloudRunnerProviderInterface } from '../services/cloud-runner-provider-interface';
|
||||
import CloudRunnerSecret from '../services/cloud-runner-secret';
|
||||
import { ProviderInterface } from '../provider-interface';
|
||||
import CloudRunnerSecret from '../../services/cloud-runner-secret';
|
||||
import KubernetesStorage from './kubernetes-storage';
|
||||
import CloudRunnerEnvironmentVariable from '../services/cloud-runner-environment-variable';
|
||||
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
|
||||
import KubernetesTaskRunner from './kubernetes-task-runner';
|
||||
import KubernetesSecret from './kubernetes-secret';
|
||||
import waitUntil from 'async-wait-until';
|
||||
import KubernetesJobSpecFactory from './kubernetes-job-spec-factory';
|
||||
import KubernetesServiceAccount from './kubernetes-service-account';
|
||||
import CloudRunnerLogger from '../services/cloud-runner-logger';
|
||||
import CloudRunnerLogger from '../../services/cloud-runner-logger';
|
||||
import { CoreV1Api } from '@kubernetes/client-node';
|
||||
import DependencyOverrideService from '../../services/depdency-override-service';
|
||||
|
||||
class Kubernetes implements CloudRunnerProviderInterface {
|
||||
class Kubernetes implements ProviderInterface {
|
||||
private kubeConfig: k8s.KubeConfig;
|
||||
private kubeClient: k8s.CoreV1Api;
|
||||
private kubeClientBatch: k8s.BatchV1Api;
|
||||
@ -38,7 +39,7 @@ class Kubernetes implements CloudRunnerProviderInterface {
|
||||
this.namespace = 'default';
|
||||
this.buildParameters = buildParameters;
|
||||
}
|
||||
public async setupSharedResources(
|
||||
public async setup(
|
||||
buildGuid: string,
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
@ -50,6 +51,9 @@ class Kubernetes implements CloudRunnerProviderInterface {
|
||||
this.pvcName = `unity-builder-pvc-${buildGuid}`;
|
||||
this.cleanupCronJobName = `unity-builder-cronjob-${buildGuid}`;
|
||||
this.serviceAccountName = `service-account-${buildGuid}`;
|
||||
if (await DependencyOverrideService.CheckHealth()) {
|
||||
await DependencyOverrideService.TryStartDependencies();
|
||||
}
|
||||
await KubernetesStorage.createPersistentVolumeClaim(
|
||||
buildParameters,
|
||||
this.pvcName,
|
||||
@ -170,7 +174,7 @@ class Kubernetes implements CloudRunnerProviderInterface {
|
||||
} catch {}
|
||||
}
|
||||
|
||||
async cleanupSharedResources(
|
||||
async cleanup(
|
||||
buildGuid: string,
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
@ -1,9 +1,9 @@
|
||||
import { V1EnvVar, V1EnvVarSource, V1SecretKeySelector } from '@kubernetes/client-node';
|
||||
import BuildParameters from '../../build-parameters';
|
||||
import { CloudRunnerBuildCommandProcessor } from '../services/cloud-runner-build-command-process';
|
||||
import CloudRunnerEnvironmentVariable from '../services/cloud-runner-environment-variable';
|
||||
import CloudRunnerSecret from '../services/cloud-runner-secret';
|
||||
import { CloudRunnerState } from '../state/cloud-runner-state';
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import { CloudRunnerBuildCommandProcessor } from '../../services/cloud-runner-build-command-process';
|
||||
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
|
||||
import CloudRunnerSecret from '../../services/cloud-runner-secret';
|
||||
import CloudRunner from '../../cloud-runner';
|
||||
|
||||
class KubernetesJobSpecFactory {
|
||||
static getJobSpec(
|
||||
@ -103,7 +103,7 @@ class KubernetesJobSpecFactory {
|
||||
name: 'main',
|
||||
image,
|
||||
command: ['/bin/sh'],
|
||||
args: ['-c', CloudRunnerBuildCommandProcessor.ProcessCommands(command, CloudRunnerState.buildParams)],
|
||||
args: ['-c', CloudRunnerBuildCommandProcessor.ProcessCommands(command, CloudRunner.buildParameters)],
|
||||
|
||||
workingDir: `${workingDirectory}`,
|
||||
resources: {
|
||||
@ -1,5 +1,5 @@
|
||||
import { CoreV1Api } from '@kubernetes/client-node';
|
||||
import CloudRunnerSecret from '../services/cloud-runner-secret';
|
||||
import CloudRunnerSecret from '../../services/cloud-runner-secret';
|
||||
import * as k8s from '@kubernetes/client-node';
|
||||
const base64 = require('base-64');
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
import waitUntil from 'async-wait-until';
|
||||
import * as core from '@actions/core';
|
||||
import * as k8s from '@kubernetes/client-node';
|
||||
import BuildParameters from '../../build-parameters';
|
||||
import CloudRunnerLogger from '../services/cloud-runner-logger';
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import CloudRunnerLogger from '../../services/cloud-runner-logger';
|
||||
import YAML from 'yaml';
|
||||
|
||||
class KubernetesStorage {
|
||||
@ -24,7 +24,9 @@ class KubernetesStorage {
|
||||
CloudRunnerLogger.log(JSON.stringify(pvcList, undefined, 4));
|
||||
if (pvcList.includes(pvcName)) {
|
||||
CloudRunnerLogger.log(`pvc ${pvcName} already exists`);
|
||||
core.setOutput('volume', pvcName);
|
||||
if (!buildParameters.isCliMode) {
|
||||
core.setOutput('volume', pvcName);
|
||||
}
|
||||
return;
|
||||
}
|
||||
CloudRunnerLogger.log(`Creating PVC ${pvcName} (does not exist)`);
|
||||
@ -48,10 +50,10 @@ class KubernetesStorage {
|
||||
CloudRunnerLogger.log(`${await this.getPVCPhase(kubeClient, name, namespace)}`);
|
||||
await waitUntil(
|
||||
async () => {
|
||||
return (await this.getPVCPhase(kubeClient, name, namespace)) !== 'Pending';
|
||||
return (await this.getPVCPhase(kubeClient, name, namespace)) === 'Pending';
|
||||
},
|
||||
{
|
||||
timeout: 500000,
|
||||
timeout: 750000,
|
||||
intervalBetweenAttempts: 15000,
|
||||
},
|
||||
);
|
||||
@ -83,7 +85,7 @@ class KubernetesStorage {
|
||||
};
|
||||
pvc.spec = {
|
||||
accessModes: ['ReadWriteOnce'],
|
||||
storageClassName: process.env.K8s_STORAGE_CLASS || 'standard',
|
||||
storageClassName: buildParameters.kubeStorageClass === '' ? 'standard' : buildParameters.kubeStorageClass,
|
||||
resources: {
|
||||
requests: {
|
||||
storage: buildParameters.kubeVolumeSize,
|
||||
@ -1,10 +1,10 @@
|
||||
import { CoreV1Api, KubeConfig, Log } from '@kubernetes/client-node';
|
||||
import { Writable } from 'stream';
|
||||
import CloudRunnerLogger from '../services/cloud-runner-logger';
|
||||
import CloudRunnerLogger from '../../services/cloud-runner-logger';
|
||||
import * as core from '@actions/core';
|
||||
import { CloudRunnerStatics } from '../cloud-runner-statics';
|
||||
import { CloudRunnerStatics } from '../../cloud-runner-statics';
|
||||
import waitUntil from 'async-wait-until';
|
||||
import { Input } from '../..';
|
||||
import CloudRunner from '../../cloud-runner';
|
||||
|
||||
class KubernetesTaskRunner {
|
||||
static async runTask(
|
||||
@ -24,7 +24,7 @@ class KubernetesTaskRunner {
|
||||
didStreamAnyLogs = true;
|
||||
let message = chunk.toString().trimRight(`\n`);
|
||||
message = `[${CloudRunnerStatics.logPrefix}] ${message}`;
|
||||
if (Input.cloudRunnerTests) {
|
||||
if (CloudRunner.buildParameters.cloudRunnerIntegrationTests) {
|
||||
output += message;
|
||||
}
|
||||
logCallback(message);
|
||||
48
src/model/cloud-runner/providers/local-docker/index.ts
Normal file
48
src/model/cloud-runner/providers/local-docker/index.ts
Normal file
@ -0,0 +1,48 @@
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import { CloudRunnerSystem } from '../../services/cloud-runner-system';
|
||||
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
|
||||
import CloudRunnerLogger from '../../services/cloud-runner-logger';
|
||||
import { ProviderInterface } from '../provider-interface';
|
||||
import CloudRunnerSecret from '../../services/cloud-runner-secret';
|
||||
|
||||
class LocalDockerCloudRunner implements ProviderInterface {
|
||||
cleanup(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {}
|
||||
setup(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {}
|
||||
public runTask(
|
||||
commands: string,
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
image: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
mountdir: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
workingdir: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
environment: CloudRunnerEnvironmentVariable[],
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
secrets: CloudRunnerSecret[],
|
||||
): Promise<string> {
|
||||
CloudRunnerLogger.log(buildGuid);
|
||||
CloudRunnerLogger.log(commands);
|
||||
return CloudRunnerSystem.Run(commands, false, false);
|
||||
}
|
||||
}
|
||||
export default LocalDockerCloudRunner;
|
||||
48
src/model/cloud-runner/providers/local/index.ts
Normal file
48
src/model/cloud-runner/providers/local/index.ts
Normal file
@ -0,0 +1,48 @@
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import { CloudRunnerSystem } from '../../services/cloud-runner-system';
|
||||
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
|
||||
import CloudRunnerLogger from '../../services/cloud-runner-logger';
|
||||
import { ProviderInterface } from '../provider-interface';
|
||||
import CloudRunnerSecret from '../../services/cloud-runner-secret';
|
||||
|
||||
class LocalCloudRunner implements ProviderInterface {
|
||||
cleanup(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {}
|
||||
public setup(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {}
|
||||
public async runTask(
|
||||
buildGuid: string,
|
||||
image: string,
|
||||
commands: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
mountdir: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
workingdir: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
environment: CloudRunnerEnvironmentVariable[],
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
secrets: CloudRunnerSecret[],
|
||||
): Promise<string> {
|
||||
CloudRunnerLogger.log(image);
|
||||
CloudRunnerLogger.log(buildGuid);
|
||||
CloudRunnerLogger.log(commands);
|
||||
return await CloudRunnerSystem.Run(commands);
|
||||
}
|
||||
}
|
||||
export default LocalCloudRunner;
|
||||
@ -1,9 +1,9 @@
|
||||
import BuildParameters from '../../build-parameters';
|
||||
import CloudRunnerEnvironmentVariable from './cloud-runner-environment-variable';
|
||||
import CloudRunnerSecret from './cloud-runner-secret';
|
||||
import CloudRunnerEnvironmentVariable from '../services/cloud-runner-environment-variable';
|
||||
import CloudRunnerSecret from '../services/cloud-runner-secret';
|
||||
|
||||
export interface CloudRunnerProviderInterface {
|
||||
cleanupSharedResources(
|
||||
export interface ProviderInterface {
|
||||
cleanup(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
@ -13,7 +13,7 @@ export interface CloudRunnerProviderInterface {
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
);
|
||||
setupSharedResources(
|
||||
setup(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
49
src/model/cloud-runner/providers/test/index.ts
Normal file
49
src/model/cloud-runner/providers/test/index.ts
Normal file
@ -0,0 +1,49 @@
|
||||
import BuildParameters from '../../../build-parameters';
|
||||
import CloudRunnerEnvironmentVariable from '../../services/cloud-runner-environment-variable';
|
||||
import CloudRunnerLogger from '../../services/cloud-runner-logger';
|
||||
import { ProviderInterface } from '../provider-interface';
|
||||
import CloudRunnerSecret from '../../services/cloud-runner-secret';
|
||||
|
||||
class TestCloudRunner implements ProviderInterface {
|
||||
cleanup(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {}
|
||||
setup(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildGuid: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
buildParameters: BuildParameters,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
branchName: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
|
||||
) {}
|
||||
public async runTask(
|
||||
commands: string,
|
||||
buildGuid: string,
|
||||
image: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
mountdir: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
workingdir: string,
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
environment: CloudRunnerEnvironmentVariable[],
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
secrets: CloudRunnerSecret[],
|
||||
): Promise<string> {
|
||||
CloudRunnerLogger.log(image);
|
||||
CloudRunnerLogger.log(buildGuid);
|
||||
CloudRunnerLogger.log(commands);
|
||||
return await new Promise((result) => {
|
||||
result(commands);
|
||||
});
|
||||
}
|
||||
}
|
||||
export default TestCloudRunner;
|
||||
63
src/model/cloud-runner/remote-client/caching.test.ts
Normal file
63
src/model/cloud-runner/remote-client/caching.test.ts
Normal file
@ -0,0 +1,63 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import BuildParameters from '../../build-parameters';
|
||||
import { Cli } from '../../cli/cli';
|
||||
import Input from '../../input';
|
||||
import UnityVersioning from '../../unity-versioning';
|
||||
import CloudRunner from '../cloud-runner';
|
||||
import { CloudRunnerSystem } from '../services/cloud-runner-system';
|
||||
import { Caching } from './caching';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
|
||||
describe('Cloud Runner Caching', () => {
|
||||
it('responds', () => {});
|
||||
});
|
||||
describe('Cloud Runner Caching', () => {
|
||||
if (process.platform === 'linux') {
|
||||
it('Simple caching works', async () => {
|
||||
Cli.options = {
|
||||
versioning: 'None',
|
||||
projectPath: 'test-project',
|
||||
unityVersion: UnityVersioning.read('test-project'),
|
||||
targetPlatform: 'StandaloneLinux64',
|
||||
cacheKey: `test-case-${uuidv4()}`,
|
||||
};
|
||||
Input.githubInputEnabled = false;
|
||||
const buildParameter = await BuildParameters.create();
|
||||
CloudRunner.buildParameters = buildParameter;
|
||||
|
||||
// create test folder
|
||||
const testFolder = path.resolve(__dirname, Cli.options.cacheKey);
|
||||
fs.mkdirSync(testFolder);
|
||||
|
||||
// crate cache folder
|
||||
const cacheFolder = path.resolve(__dirname, `cache-${Cli.options.cacheKey}`);
|
||||
fs.mkdirSync(cacheFolder);
|
||||
|
||||
// add test has file to test folders
|
||||
fs.writeFileSync(path.resolve(testFolder, 'test.txt'), Cli.options.cacheKey);
|
||||
await Caching.PushToCache(cacheFolder, testFolder, `${Cli.options.cacheKey}`);
|
||||
|
||||
// delete test folder
|
||||
fs.rmdirSync(testFolder, { recursive: true });
|
||||
await Caching.PullFromCache(
|
||||
cacheFolder.replace(/\\/g, `/`),
|
||||
testFolder.replace(/\\/g, `/`),
|
||||
`${Cli.options.cacheKey}`,
|
||||
);
|
||||
await CloudRunnerSystem.Run(`du -h ${__dirname}`);
|
||||
await CloudRunnerSystem.Run(`tree ${testFolder}`);
|
||||
await CloudRunnerSystem.Run(`tree ${cacheFolder}`);
|
||||
|
||||
// compare validity to original hash
|
||||
expect(fs.readFileSync(path.resolve(testFolder, 'test.txt'), { encoding: 'utf8' }).toString()).toContain(
|
||||
Cli.options.cacheKey,
|
||||
);
|
||||
fs.rmdirSync(testFolder, { recursive: true });
|
||||
fs.rmdirSync(cacheFolder, { recursive: true });
|
||||
|
||||
Input.githubInputEnabled = true;
|
||||
delete Cli.options;
|
||||
}, 1000000);
|
||||
}
|
||||
});
|
||||
171
src/model/cloud-runner/remote-client/caching.ts
Normal file
171
src/model/cloud-runner/remote-client/caching.ts
Normal file
@ -0,0 +1,171 @@
|
||||
import { assert } from 'console';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import CloudRunner from '../cloud-runner';
|
||||
import CloudRunnerLogger from '../services/cloud-runner-logger';
|
||||
import { CloudRunnerFolders } from '../services/cloud-runner-folders';
|
||||
import { CloudRunnerSystem } from '../services/cloud-runner-system';
|
||||
import { LfsHashing } from '../services/lfs-hashing';
|
||||
import { RemoteClientLogger } from './remote-client-logger';
|
||||
import { Cli } from '../../cli/cli';
|
||||
import { CliFunction } from '../../cli/cli-functions-repository';
|
||||
// eslint-disable-next-line github/no-then
|
||||
const fileExists = async (fpath) => !!(await fs.promises.stat(fpath).catch(() => false));
|
||||
|
||||
export class Caching {
|
||||
@CliFunction(`cache-push`, `push to cache`)
|
||||
static async cachePush() {
|
||||
try {
|
||||
const buildParameter = JSON.parse(process.env.BUILD_PARAMETERS || '{}');
|
||||
CloudRunner.buildParameters = buildParameter;
|
||||
await Caching.PushToCache(
|
||||
Cli.options['cachePushTo'],
|
||||
Cli.options['cachePushFrom'],
|
||||
Cli.options['artifactName'] || '',
|
||||
);
|
||||
} catch (error: any) {
|
||||
CloudRunnerLogger.log(`${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
@CliFunction(`cache-pull`, `pull from cache`)
|
||||
static async cachePull() {
|
||||
try {
|
||||
const buildParameter = JSON.parse(process.env.BUILD_PARAMETERS || '{}');
|
||||
CloudRunner.buildParameters = buildParameter;
|
||||
await Caching.PullFromCache(
|
||||
Cli.options['cachePushFrom'],
|
||||
Cli.options['cachePushTo'],
|
||||
Cli.options['artifactName'] || '',
|
||||
);
|
||||
} catch (error: any) {
|
||||
CloudRunnerLogger.log(`${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
public static async PushToCache(cacheFolder: string, sourceFolder: string, cacheArtifactName: string) {
|
||||
cacheArtifactName = cacheArtifactName.replace(' ', '');
|
||||
const startPath = process.cwd();
|
||||
try {
|
||||
if (!(await fileExists(cacheFolder))) {
|
||||
await CloudRunnerSystem.Run(`mkdir -p ${cacheFolder}`);
|
||||
}
|
||||
process.chdir(path.resolve(sourceFolder, '..'));
|
||||
|
||||
if (CloudRunner.buildParameters.cloudRunnerIntegrationTests) {
|
||||
CloudRunnerLogger.log(
|
||||
`Hashed cache folder ${await LfsHashing.hashAllFiles(sourceFolder)} ${sourceFolder} ${path.basename(
|
||||
sourceFolder,
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
// eslint-disable-next-line func-style
|
||||
const formatFunction = function (format: string) {
|
||||
const arguments_ = Array.prototype.slice.call(
|
||||
[path.resolve(sourceFolder, '..'), cacheFolder, cacheArtifactName],
|
||||
1,
|
||||
);
|
||||
return format.replace(/{(\d+)}/g, function (match, number) {
|
||||
return typeof arguments_[number] != 'undefined' ? arguments_[number] : match;
|
||||
});
|
||||
};
|
||||
await CloudRunnerSystem.Run(`zip -q -r ${cacheArtifactName}.zip ${path.basename(sourceFolder)}`);
|
||||
assert(await fileExists(`${cacheArtifactName}.zip`), 'cache zip exists');
|
||||
assert(await fileExists(path.basename(sourceFolder)), 'source folder exists');
|
||||
if (CloudRunner.buildParameters.cachePushOverrideCommand) {
|
||||
await CloudRunnerSystem.Run(formatFunction(CloudRunner.buildParameters.cachePushOverrideCommand));
|
||||
}
|
||||
await CloudRunnerSystem.Run(`mv ${cacheArtifactName}.zip ${cacheFolder}`);
|
||||
RemoteClientLogger.log(`moved ${cacheArtifactName}.zip to ${cacheFolder}`);
|
||||
assert(
|
||||
await fileExists(`${path.join(cacheFolder, cacheArtifactName)}.zip`),
|
||||
'cache zip exists inside cache folder',
|
||||
);
|
||||
} catch (error) {
|
||||
process.chdir(`${startPath}`);
|
||||
throw error;
|
||||
}
|
||||
process.chdir(`${startPath}`);
|
||||
}
|
||||
public static async PullFromCache(cacheFolder: string, destinationFolder: string, cacheArtifactName: string = ``) {
|
||||
cacheArtifactName = cacheArtifactName.replace(' ', '');
|
||||
const startPath = process.cwd();
|
||||
RemoteClientLogger.log(`Caching for ${path.basename(destinationFolder)}`);
|
||||
try {
|
||||
if (!(await fileExists(cacheFolder))) {
|
||||
await fs.promises.mkdir(cacheFolder);
|
||||
}
|
||||
|
||||
if (!(await fileExists(destinationFolder))) {
|
||||
await fs.promises.mkdir(destinationFolder);
|
||||
}
|
||||
|
||||
const latestInBranch = await (await CloudRunnerSystem.Run(`ls -t "${cacheFolder}" | grep .zip$ | head -1`))
|
||||
.replace(/\n/g, ``)
|
||||
.replace('.zip', '');
|
||||
|
||||
process.chdir(cacheFolder);
|
||||
|
||||
const cacheSelection =
|
||||
cacheArtifactName !== `` && (await fileExists(`${cacheArtifactName}.zip`)) ? cacheArtifactName : latestInBranch;
|
||||
await CloudRunnerLogger.log(`cache key ${cacheArtifactName} selection ${cacheSelection}`);
|
||||
|
||||
// eslint-disable-next-line func-style
|
||||
const formatFunction = function (format: string) {
|
||||
const arguments_ = Array.prototype.slice.call(
|
||||
[path.resolve(destinationFolder, '..'), cacheFolder, cacheArtifactName],
|
||||
1,
|
||||
);
|
||||
return format.replace(/{(\d+)}/g, function (match, number) {
|
||||
return typeof arguments_[number] != 'undefined' ? arguments_[number] : match;
|
||||
});
|
||||
};
|
||||
|
||||
if (CloudRunner.buildParameters.cachePullOverrideCommand) {
|
||||
await CloudRunnerSystem.Run(formatFunction(CloudRunner.buildParameters.cachePullOverrideCommand));
|
||||
}
|
||||
|
||||
if (await fileExists(`${cacheSelection}.zip`)) {
|
||||
const resultsFolder = `results${CloudRunner.buildParameters.buildGuid}`;
|
||||
await CloudRunnerSystem.Run(`mkdir -p ${resultsFolder}`);
|
||||
RemoteClientLogger.log(`cache item exists ${cacheFolder}/${cacheSelection}.zip`);
|
||||
const fullResultsFolder = path.join(cacheFolder, resultsFolder);
|
||||
await CloudRunnerSystem.Run(`unzip -q ${cacheSelection}.zip -d ${path.basename(resultsFolder)}`);
|
||||
RemoteClientLogger.log(`cache item extracted to ${fullResultsFolder}`);
|
||||
assert(await fileExists(fullResultsFolder), `cache extraction results folder exists`);
|
||||
const destinationParentFolder = path.resolve(destinationFolder, '..');
|
||||
|
||||
if (await fileExists(destinationFolder)) {
|
||||
await fs.promises.rmdir(destinationFolder, { recursive: true });
|
||||
}
|
||||
await CloudRunnerSystem.Run(
|
||||
`mv "${path.join(fullResultsFolder, path.basename(destinationFolder))}" "${destinationParentFolder}"`,
|
||||
);
|
||||
await CloudRunnerSystem.Run(`du -sh ${path.join(destinationParentFolder, path.basename(destinationFolder))}`);
|
||||
const contents = await fs.promises.readdir(
|
||||
path.join(destinationParentFolder, path.basename(destinationFolder)),
|
||||
);
|
||||
CloudRunnerLogger.log(
|
||||
`There is ${contents.length} files/dir in the cache pulled contents for ${path.basename(destinationFolder)}`,
|
||||
);
|
||||
} else {
|
||||
RemoteClientLogger.logWarning(`cache item ${cacheArtifactName} doesn't exist ${destinationFolder}`);
|
||||
if (cacheSelection !== ``) {
|
||||
RemoteClientLogger.logWarning(`cache item ${cacheArtifactName}.zip doesn't exist ${destinationFolder}`);
|
||||
throw new Error(`Failed to get cache item, but cache hit was found: ${cacheSelection}`);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
process.chdir(`${startPath}`);
|
||||
throw error;
|
||||
}
|
||||
process.chdir(`${startPath}`);
|
||||
}
|
||||
|
||||
public static async handleCachePurging() {
|
||||
if (process.env.PURGE_REMOTE_BUILDER_CACHE !== undefined) {
|
||||
RemoteClientLogger.log(`purging ${CloudRunnerFolders.purgeRemoteCaching}`);
|
||||
fs.promises.rmdir(CloudRunnerFolders.cacheFolder, { recursive: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
95
src/model/cloud-runner/remote-client/index.ts
Normal file
95
src/model/cloud-runner/remote-client/index.ts
Normal file
@ -0,0 +1,95 @@
|
||||
import fs from 'fs';
|
||||
import CloudRunner from '../cloud-runner';
|
||||
import { CloudRunnerFolders } from '../services/cloud-runner-folders';
|
||||
import { Caching } from './caching';
|
||||
import { LfsHashing } from '../services/lfs-hashing';
|
||||
import { RemoteClientLogger } from './remote-client-logger';
|
||||
import path from 'path';
|
||||
import { assert } from 'console';
|
||||
import CloudRunnerLogger from '../services/cloud-runner-logger';
|
||||
import { CliFunction } from '../../cli/cli-functions-repository';
|
||||
import { CloudRunnerSystem } from '../services/cloud-runner-system';
|
||||
|
||||
export class RemoteClient {
|
||||
public static async bootstrapRepository() {
|
||||
try {
|
||||
await CloudRunnerSystem.Run(`mkdir -p ${CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute}`);
|
||||
await CloudRunnerSystem.Run(`mkdir -p ${CloudRunnerFolders.repoPathAbsolute}`);
|
||||
await CloudRunnerSystem.Run(`mkdir -p ${CloudRunnerFolders.cacheFolderFull}`);
|
||||
process.chdir(CloudRunnerFolders.repoPathAbsolute);
|
||||
await RemoteClient.cloneRepoWithoutLFSFiles();
|
||||
await RemoteClient.sizeOfFolder('repo before lfs cache pull', CloudRunnerFolders.repoPathAbsolute);
|
||||
const lfsHashes = await LfsHashing.createLFSHashFiles();
|
||||
if (fs.existsSync(CloudRunnerFolders.libraryFolderAbsolute)) {
|
||||
RemoteClientLogger.logWarning(`!Warning!: The Unity library was included in the git repository`);
|
||||
}
|
||||
await Caching.PullFromCache(
|
||||
CloudRunnerFolders.lfsCacheFolderFull,
|
||||
CloudRunnerFolders.lfsFolderAbsolute,
|
||||
`${lfsHashes.lfsGuidSum}`,
|
||||
);
|
||||
await RemoteClient.sizeOfFolder('repo after lfs cache pull', CloudRunnerFolders.repoPathAbsolute);
|
||||
await RemoteClient.pullLatestLFS();
|
||||
await RemoteClient.sizeOfFolder('repo before lfs git pull', CloudRunnerFolders.repoPathAbsolute);
|
||||
await Caching.PushToCache(
|
||||
CloudRunnerFolders.lfsCacheFolderFull,
|
||||
CloudRunnerFolders.lfsFolderAbsolute,
|
||||
`${lfsHashes.lfsGuidSum}`,
|
||||
);
|
||||
await Caching.PullFromCache(CloudRunnerFolders.libraryCacheFolderFull, CloudRunnerFolders.libraryFolderAbsolute);
|
||||
await RemoteClient.sizeOfFolder('repo after library cache pull', CloudRunnerFolders.repoPathAbsolute);
|
||||
await Caching.handleCachePurging();
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private static async sizeOfFolder(message: string, folder: string) {
|
||||
CloudRunnerLogger.log(`Size of ${message}`);
|
||||
await CloudRunnerSystem.Run(`du -sh ${folder}`);
|
||||
}
|
||||
|
||||
private static async cloneRepoWithoutLFSFiles() {
|
||||
try {
|
||||
process.chdir(`${CloudRunnerFolders.repoPathAbsolute}`);
|
||||
RemoteClientLogger.log(`Initializing source repository for cloning with caching of LFS files`);
|
||||
await CloudRunnerSystem.Run(`git config --global advice.detachedHead false`);
|
||||
RemoteClientLogger.log(`Cloning the repository being built:`);
|
||||
await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge --skip -- %f"`);
|
||||
await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process --skip"`);
|
||||
await CloudRunnerSystem.Run(
|
||||
`git clone -q ${CloudRunnerFolders.targetBuildRepoUrl} ${path.resolve(
|
||||
`..`,
|
||||
path.basename(CloudRunnerFolders.repoPathAbsolute),
|
||||
)}`,
|
||||
);
|
||||
await CloudRunnerSystem.Run(`git lfs install`);
|
||||
assert(fs.existsSync(`.git`), 'git folder exists');
|
||||
RemoteClientLogger.log(`${CloudRunner.buildParameters.branch}`);
|
||||
await CloudRunnerSystem.Run(`git checkout ${CloudRunner.buildParameters.branch}`);
|
||||
assert(fs.existsSync(path.join(`.git`, `lfs`)), 'LFS folder should not exist before caching');
|
||||
RemoteClientLogger.log(`Checked out ${process.env.GITHUB_SHA}`);
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private static async pullLatestLFS() {
|
||||
process.chdir(CloudRunnerFolders.repoPathAbsolute);
|
||||
await CloudRunnerSystem.Run(`git config --global filter.lfs.smudge "git-lfs smudge -- %f"`);
|
||||
await CloudRunnerSystem.Run(`git config --global filter.lfs.process "git-lfs filter-process"`);
|
||||
await CloudRunnerSystem.Run(`git lfs pull`);
|
||||
RemoteClientLogger.log(`pulled latest LFS files`);
|
||||
assert(fs.existsSync(CloudRunnerFolders.lfsFolderAbsolute));
|
||||
}
|
||||
|
||||
@CliFunction(`remote-cli`, `sets up a repository, usually before a game-ci build`)
|
||||
static async runRemoteClientJob() {
|
||||
const buildParameter = JSON.parse(process.env.BUILD_PARAMETERS || '{}');
|
||||
RemoteClientLogger.log(`Build Params:
|
||||
${JSON.stringify(buildParameter, undefined, 4)}
|
||||
`);
|
||||
CloudRunner.buildParameters = buildParameter;
|
||||
await RemoteClient.bootstrapRepository();
|
||||
}
|
||||
}
|
||||
@ -1,4 +1,4 @@
|
||||
import CloudRunnerLogger from '../../../cloud-runner/services/cloud-runner-logger';
|
||||
import CloudRunnerLogger from '../services/cloud-runner-logger';
|
||||
|
||||
export class RemoteClientLogger {
|
||||
public static log(message: string) {
|
||||
@ -1,25 +1,27 @@
|
||||
import { BuildParameters, Input } from '../..';
|
||||
import { BuildParameters } from '../..';
|
||||
import YAML from 'yaml';
|
||||
import CloudRunnerSecret from './cloud-runner-secret';
|
||||
import CloudRunner from '../cloud-runner';
|
||||
|
||||
export class CloudRunnerBuildCommandProcessor {
|
||||
public static ProcessCommands(commands: string, buildParameters: BuildParameters): string {
|
||||
const hooks = CloudRunnerBuildCommandProcessor.getHooks().filter((x) => x.step.includes(`all`));
|
||||
const hooks = CloudRunnerBuildCommandProcessor.getHooks(buildParameters.customJobHooks).filter((x) =>
|
||||
x.step.includes(`all`),
|
||||
);
|
||||
|
||||
return `echo "---"
|
||||
echo "start cloud runner init"
|
||||
${Input.cloudRunnerTests ? '' : '#'} printenv
|
||||
echo "start cloud runner job"
|
||||
${CloudRunner.buildParameters.cloudRunnerIntegrationTests ? '' : '#'} printenv
|
||||
echo "start of cloud runner job"
|
||||
${hooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
|
||||
${commands}
|
||||
${hooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
|
||||
echo "end of cloud runner job
|
||||
---${buildParameters.logId}"
|
||||
`;
|
||||
echo "end of cloud runner job"
|
||||
echo "---${buildParameters.logId}"`;
|
||||
}
|
||||
|
||||
public static getHooks(): Hook[] {
|
||||
const experimentHooks = process.env.EXPERIMENTAL_HOOKS;
|
||||
public static getHooks(customJobHooks): Hook[] {
|
||||
const experimentHooks = customJobHooks;
|
||||
let output = new Array<Hook>();
|
||||
if (experimentHooks && experimentHooks !== '') {
|
||||
try {
|
||||
|
||||
73
src/model/cloud-runner/services/cloud-runner-folders.ts
Normal file
73
src/model/cloud-runner/services/cloud-runner-folders.ts
Normal file
@ -0,0 +1,73 @@
|
||||
import path from 'path';
|
||||
import { CloudRunner } from '../..';
|
||||
|
||||
export class CloudRunnerFolders {
|
||||
public static readonly repositoryFolder = 'repo';
|
||||
|
||||
// only the following paths that do not start a path.join with another "Full" suffixed property need to start with an absolute /
|
||||
|
||||
public static get uniqueCloudRunnerJobFolderAbsolute(): string {
|
||||
return path.join(`/`, CloudRunnerFolders.buildVolumeFolder, CloudRunner.buildParameters.buildGuid);
|
||||
}
|
||||
|
||||
public static get cacheFolderFull(): string {
|
||||
return path.join(
|
||||
'/',
|
||||
CloudRunnerFolders.buildVolumeFolder,
|
||||
CloudRunnerFolders.cacheFolder,
|
||||
CloudRunner.buildParameters.cacheKey,
|
||||
);
|
||||
}
|
||||
|
||||
public static get builderPathAbsolute(): string {
|
||||
return path.join(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute, `builder`);
|
||||
}
|
||||
|
||||
public static get repoPathAbsolute(): string {
|
||||
return path.join(CloudRunnerFolders.uniqueCloudRunnerJobFolderAbsolute, CloudRunnerFolders.repositoryFolder);
|
||||
}
|
||||
|
||||
public static get projectPathAbsolute(): string {
|
||||
return path.join(CloudRunnerFolders.repoPathAbsolute, CloudRunner.buildParameters.projectPath);
|
||||
}
|
||||
|
||||
public static get libraryFolderAbsolute(): string {
|
||||
return path.join(CloudRunnerFolders.projectPathAbsolute, `Library`);
|
||||
}
|
||||
|
||||
public static get projectBuildFolderAbsolute(): string {
|
||||
return path.join(CloudRunnerFolders.repoPathAbsolute, CloudRunner.buildParameters.buildPath);
|
||||
}
|
||||
|
||||
public static get lfsFolderAbsolute(): string {
|
||||
return path.join(CloudRunnerFolders.repoPathAbsolute, `.git`, `lfs`);
|
||||
}
|
||||
|
||||
public static get purgeRemoteCaching(): boolean {
|
||||
return process.env.PURGE_REMOTE_BUILDER_CACHE !== undefined;
|
||||
}
|
||||
|
||||
public static get lfsCacheFolderFull() {
|
||||
return path.join(CloudRunnerFolders.cacheFolderFull, `lfs`);
|
||||
}
|
||||
|
||||
public static get libraryCacheFolderFull() {
|
||||
return path.join(CloudRunnerFolders.cacheFolderFull, `Library`);
|
||||
}
|
||||
|
||||
public static get unityBuilderRepoUrl(): string {
|
||||
return `https://${CloudRunner.buildParameters.gitPrivateToken}@github.com/game-ci/unity-builder.git`;
|
||||
}
|
||||
|
||||
public static get targetBuildRepoUrl(): string {
|
||||
return `https://${CloudRunner.buildParameters.gitPrivateToken}@github.com/${CloudRunner.buildParameters.githubRepo}.git`;
|
||||
}
|
||||
|
||||
public static get buildVolumeFolder() {
|
||||
return 'data';
|
||||
}
|
||||
|
||||
public static get cacheFolder() {
|
||||
return 'cache';
|
||||
}
|
||||
}
|
||||
@ -2,7 +2,7 @@ import { customAlphabet } from 'nanoid';
|
||||
import CloudRunnerConstants from './cloud-runner-constants';
|
||||
|
||||
class CloudRunnerNamespace {
|
||||
static generateBuildName(runNumber: string | number, platform: string) {
|
||||
static generateGuid(runNumber: string | number, platform: string) {
|
||||
const nanoid = customAlphabet(CloudRunnerConstants.alphabet, 4);
|
||||
return `${runNumber}-${platform.toLowerCase().replace('standalone', '')}-${nanoid()}`;
|
||||
}
|
||||
@ -0,0 +1,59 @@
|
||||
import Input from '../../input';
|
||||
import { GenericInputReader } from '../../input-readers/generic-input-reader';
|
||||
|
||||
const formatFunction = (value, arguments_) => {
|
||||
for (const element of arguments_) {
|
||||
value = value.replace(`{${element.key}}`, element.value);
|
||||
}
|
||||
return value;
|
||||
};
|
||||
|
||||
class CloudRunnerQueryOverride {
|
||||
static queryOverrides: any;
|
||||
|
||||
public static query(key, alternativeKey) {
|
||||
if (CloudRunnerQueryOverride.queryOverrides && CloudRunnerQueryOverride.queryOverrides[key] !== undefined) {
|
||||
return CloudRunnerQueryOverride.queryOverrides[key];
|
||||
}
|
||||
if (
|
||||
CloudRunnerQueryOverride.queryOverrides &&
|
||||
alternativeKey &&
|
||||
CloudRunnerQueryOverride.queryOverrides[alternativeKey] !== undefined
|
||||
) {
|
||||
return CloudRunnerQueryOverride.queryOverrides[alternativeKey];
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
private static shouldUseOverride(query) {
|
||||
if (Input.readInputOverrideCommand() !== '') {
|
||||
if (Input.readInputFromOverrideList() !== '') {
|
||||
const doesInclude =
|
||||
Input.readInputFromOverrideList().split(',').includes(query) ||
|
||||
Input.readInputFromOverrideList().split(',').includes(Input.ToEnvVarFormat(query));
|
||||
return doesInclude ? true : false;
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static async queryOverride(query) {
|
||||
if (!this.shouldUseOverride(query)) {
|
||||
throw new Error(`Should not be trying to run override query on ${query}`);
|
||||
}
|
||||
|
||||
return await GenericInputReader.Run(formatFunction(Input.readInputOverrideCommand(), [{ key: 0, value: query }]));
|
||||
}
|
||||
|
||||
public static async PopulateQueryOverrideInput() {
|
||||
const queries = Input.readInputFromOverrideList().split(',');
|
||||
CloudRunnerQueryOverride.queryOverrides = new Array();
|
||||
for (const element of queries) {
|
||||
if (CloudRunnerQueryOverride.shouldUseOverride(element)) {
|
||||
CloudRunnerQueryOverride.queryOverrides[element] = await CloudRunnerQueryOverride.queryOverride(element);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
export default CloudRunnerQueryOverride;
|
||||
45
src/model/cloud-runner/services/cloud-runner-system.ts
Normal file
45
src/model/cloud-runner/services/cloud-runner-system.ts
Normal file
@ -0,0 +1,45 @@
|
||||
import { exec } from 'child_process';
|
||||
import { RemoteClientLogger } from '../remote-client/remote-client-logger';
|
||||
|
||||
export class CloudRunnerSystem {
|
||||
public static async Run(command: string, suppressError = false, suppressLogs = false) {
|
||||
for (const element of command.split(`\n`)) {
|
||||
if (!suppressLogs) {
|
||||
RemoteClientLogger.log(element);
|
||||
}
|
||||
}
|
||||
return await new Promise<string>((promise, throwError) => {
|
||||
let output = '';
|
||||
const child = exec(command, (error, stdout, stderr) => {
|
||||
if (!suppressError && error) {
|
||||
RemoteClientLogger.log(error.toString());
|
||||
throwError(error);
|
||||
}
|
||||
if (stderr) {
|
||||
const diagnosticOutput = `${stderr.toString()}`;
|
||||
if (!suppressLogs) {
|
||||
RemoteClientLogger.logCliDiagnostic(diagnosticOutput);
|
||||
}
|
||||
output += diagnosticOutput;
|
||||
}
|
||||
const outputChunk = `${stdout}`;
|
||||
output += outputChunk;
|
||||
});
|
||||
child.on('close', (code) => {
|
||||
if (!suppressLogs) {
|
||||
RemoteClientLogger.log(`[${code}]`);
|
||||
}
|
||||
if (code !== 0 && !suppressError) {
|
||||
throwError(output);
|
||||
}
|
||||
const outputLines = output.split(`\n`);
|
||||
for (const element of outputLines) {
|
||||
if (!suppressLogs) {
|
||||
RemoteClientLogger.log(element);
|
||||
}
|
||||
}
|
||||
promise(output);
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
21
src/model/cloud-runner/services/depdency-override-service.ts
Normal file
21
src/model/cloud-runner/services/depdency-override-service.ts
Normal file
@ -0,0 +1,21 @@
|
||||
import Input from '../../input';
|
||||
import { CloudRunnerSystem } from './cloud-runner-system';
|
||||
|
||||
class DependencyOverrideService {
|
||||
public static async CheckHealth() {
|
||||
if (Input.checkDependencyHealthOverride) {
|
||||
try {
|
||||
await CloudRunnerSystem.Run(Input.checkDependencyHealthOverride);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
public static async TryStartDependencies() {
|
||||
if (Input.startDependenciesOverride) {
|
||||
await CloudRunnerSystem.Run(Input.startDependenciesOverride);
|
||||
}
|
||||
}
|
||||
}
|
||||
export default DependencyOverrideService;
|
||||
@ -1,12 +1,12 @@
|
||||
import path from 'path';
|
||||
import { CloudRunnerState } from '../../../cloud-runner/state/cloud-runner-state';
|
||||
import { CloudRunnerFolders } from './cloud-runner-folders';
|
||||
import { CloudRunnerSystem } from './cloud-runner-system';
|
||||
import fs from 'fs';
|
||||
import { assert } from 'console';
|
||||
import { Input } from '../../..';
|
||||
import { RemoteClientLogger } from './remote-client-logger';
|
||||
import { Cli } from '../../cli/cli';
|
||||
import { CliFunction } from '../../cli/cli-functions-repository';
|
||||
|
||||
export class LFSHashing {
|
||||
export class LfsHashing {
|
||||
public static async createLFSHashFiles() {
|
||||
try {
|
||||
await CloudRunnerSystem.Run(`git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-guid`);
|
||||
@ -15,16 +15,13 @@ export class LFSHashing {
|
||||
assert(fs.existsSync(`.lfs-assets-guid`));
|
||||
const lfsHashes = {
|
||||
lfsGuid: fs
|
||||
.readFileSync(`${path.join(CloudRunnerState.repoPathFull, `.lfs-assets-guid`)}`, 'utf8')
|
||||
.readFileSync(`${path.join(CloudRunnerFolders.repoPathAbsolute, `.lfs-assets-guid`)}`, 'utf8')
|
||||
.replace(/\n/g, ``),
|
||||
lfsGuidSum: fs
|
||||
.readFileSync(`${path.join(CloudRunnerState.repoPathFull, `.lfs-assets-guid-sum`)}`, 'utf8')
|
||||
.readFileSync(`${path.join(CloudRunnerFolders.repoPathAbsolute, `.lfs-assets-guid-sum`)}`, 'utf8')
|
||||
.replace(' .lfs-assets-guid', '')
|
||||
.replace(/\n/g, ``),
|
||||
};
|
||||
if (Input.cloudRunnerTests) {
|
||||
RemoteClientLogger.log(lfsHashes.lfsGuid);
|
||||
RemoteClientLogger.log(lfsHashes.lfsGuidSum);
|
||||
}
|
||||
return lfsHashes;
|
||||
} catch (error) {
|
||||
throw error;
|
||||
@ -39,4 +36,10 @@ export class LFSHashing {
|
||||
process.chdir(startPath);
|
||||
return result;
|
||||
}
|
||||
|
||||
@CliFunction(`hash`, `hash all folder contents`)
|
||||
static async hash() {
|
||||
const folder = Cli.options['cachePushFrom'];
|
||||
LfsHashing.hashAllFiles(folder);
|
||||
}
|
||||
}
|
||||
@ -1,24 +1,24 @@
|
||||
import { Input } from '../..';
|
||||
import { CloudRunner, Input } from '../..';
|
||||
import ImageEnvironmentFactory from '../../image-environment-factory';
|
||||
import CloudRunnerEnvironmentVariable from './cloud-runner-environment-variable';
|
||||
import { CloudRunnerState } from '../state/cloud-runner-state';
|
||||
import { CloudRunnerBuildCommandProcessor } from './cloud-runner-build-command-process';
|
||||
import CloudRunnerSecret from './cloud-runner-secret';
|
||||
import CloudRunnerQueryOverride from './cloud-runner-query-override';
|
||||
|
||||
export class TaskParameterSerializer {
|
||||
public static readBuildEnvironmentVariables(): CloudRunnerEnvironmentVariable[] {
|
||||
TaskParameterSerializer.setupDefaultSecrets();
|
||||
return [
|
||||
{
|
||||
name: 'ContainerMemory',
|
||||
value: CloudRunnerState.buildParams.cloudRunnerMemory,
|
||||
value: CloudRunner.buildParameters.cloudRunnerMemory,
|
||||
},
|
||||
{
|
||||
name: 'ContainerCpu',
|
||||
value: CloudRunnerState.buildParams.cloudRunnerCpu,
|
||||
value: CloudRunner.buildParameters.cloudRunnerCpu,
|
||||
},
|
||||
{
|
||||
name: 'BUILD_TARGET',
|
||||
value: CloudRunnerState.buildParams.targetPlatform,
|
||||
value: CloudRunner.buildParameters.targetPlatform,
|
||||
},
|
||||
...TaskParameterSerializer.serializeBuildParamsAndInput,
|
||||
];
|
||||
@ -27,7 +27,7 @@ export class TaskParameterSerializer {
|
||||
let array = new Array();
|
||||
array = TaskParameterSerializer.readBuildParameters(array);
|
||||
array = TaskParameterSerializer.readInput(array);
|
||||
const configurableHooks = CloudRunnerBuildCommandProcessor.getHooks();
|
||||
const configurableHooks = CloudRunnerBuildCommandProcessor.getHooks(CloudRunner.buildParameters.customJobHooks);
|
||||
const secrets = configurableHooks.map((x) => x.secrets).filter((x) => x !== undefined && x.length > 0);
|
||||
if (secrets.length > 0) {
|
||||
// eslint-disable-next-line unicorn/no-array-reduce
|
||||
@ -46,14 +46,14 @@ export class TaskParameterSerializer {
|
||||
}
|
||||
|
||||
private static readBuildParameters(array: any[]) {
|
||||
const keys = Object.keys(CloudRunnerState.buildParams);
|
||||
const keys = Object.keys(CloudRunner.buildParameters);
|
||||
for (const element of keys) {
|
||||
array.push({
|
||||
name: element,
|
||||
value: CloudRunnerState.buildParams[element],
|
||||
value: CloudRunner.buildParameters[element],
|
||||
});
|
||||
}
|
||||
array.push({ name: 'buildParameters', value: JSON.stringify(CloudRunnerState.buildParams) });
|
||||
array.push({ name: 'buildParameters', value: JSON.stringify(CloudRunner.buildParameters) });
|
||||
return array;
|
||||
}
|
||||
|
||||
@ -70,16 +70,40 @@ export class TaskParameterSerializer {
|
||||
return array;
|
||||
}
|
||||
|
||||
private static setupDefaultSecrets() {
|
||||
if (CloudRunnerState.defaultSecrets === undefined)
|
||||
CloudRunnerState.defaultSecrets = ImageEnvironmentFactory.getEnvironmentVariables(
|
||||
CloudRunnerState.buildParams,
|
||||
).map((x) => {
|
||||
return {
|
||||
ParameterKey: x.name,
|
||||
EnvironmentVariable: x.name,
|
||||
ParameterValue: x.value,
|
||||
};
|
||||
public static readDefaultSecrets(): CloudRunnerSecret[] {
|
||||
let array = new Array();
|
||||
array = TaskParameterSerializer.tryAddInput(array, 'UNITY_SERIAL');
|
||||
array = TaskParameterSerializer.tryAddInput(array, 'UNITY_EMAIL');
|
||||
array = TaskParameterSerializer.tryAddInput(array, 'UNITY_PASSWORD');
|
||||
array.push(
|
||||
...ImageEnvironmentFactory.getEnvironmentVariables(CloudRunner.buildParameters)
|
||||
.filter((x) => array.every((y) => y.ParameterKey !== x.name))
|
||||
.map((x) => {
|
||||
return {
|
||||
ParameterKey: x.name,
|
||||
EnvironmentVariable: x.name,
|
||||
ParameterValue: x.value,
|
||||
};
|
||||
}),
|
||||
);
|
||||
return array;
|
||||
}
|
||||
private static getValue(key) {
|
||||
return CloudRunnerQueryOverride.queryOverrides !== undefined &&
|
||||
CloudRunnerQueryOverride.queryOverrides[key] !== undefined
|
||||
? CloudRunnerQueryOverride.queryOverrides[key]
|
||||
: process.env[key];
|
||||
}
|
||||
s;
|
||||
private static tryAddInput(array, key): CloudRunnerSecret[] {
|
||||
const value = TaskParameterSerializer.getValue(key);
|
||||
if (value !== undefined && value !== '') {
|
||||
array.push({
|
||||
ParameterKey: key,
|
||||
EnvironmentVariable: key,
|
||||
ParameterValue: value,
|
||||
});
|
||||
}
|
||||
return array;
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,81 +0,0 @@
|
||||
import path from 'path';
|
||||
import { BuildParameters } from '../..';
|
||||
import { CloudRunnerProviderInterface } from '../services/cloud-runner-provider-interface';
|
||||
import CloudRunnerSecret from '../services/cloud-runner-secret';
|
||||
|
||||
export class CloudRunnerState {
|
||||
public static CloudRunnerProviderPlatform: CloudRunnerProviderInterface;
|
||||
public static buildParams: BuildParameters;
|
||||
public static defaultSecrets: CloudRunnerSecret[];
|
||||
public static readonly repositoryFolder = 'repo';
|
||||
|
||||
// only the following paths that do not start a path.join with another "Full" suffixed property need to start with an absolute /
|
||||
|
||||
public static get buildPathFull(): string {
|
||||
return path.join(`/`, CloudRunnerState.buildVolumeFolder, CloudRunnerState.buildParams.buildGuid);
|
||||
}
|
||||
|
||||
public static get cacheFolderFull(): string {
|
||||
return path.join(
|
||||
'/',
|
||||
CloudRunnerState.buildVolumeFolder,
|
||||
CloudRunnerState.cacheFolder,
|
||||
CloudRunnerState.branchName,
|
||||
);
|
||||
}
|
||||
|
||||
static setup(buildParameters: BuildParameters) {
|
||||
CloudRunnerState.buildParams = buildParameters;
|
||||
}
|
||||
|
||||
public static get branchName(): string {
|
||||
return CloudRunnerState.buildParams.branch;
|
||||
}
|
||||
public static get builderPathFull(): string {
|
||||
return path.join(CloudRunnerState.buildPathFull, `builder`);
|
||||
}
|
||||
|
||||
public static get repoPathFull(): string {
|
||||
return path.join(CloudRunnerState.buildPathFull, CloudRunnerState.repositoryFolder);
|
||||
}
|
||||
|
||||
public static get projectPathFull(): string {
|
||||
return path.join(CloudRunnerState.repoPathFull, CloudRunnerState.buildParams.projectPath);
|
||||
}
|
||||
|
||||
public static get libraryFolderFull(): string {
|
||||
return path.join(CloudRunnerState.projectPathFull, `Library`);
|
||||
}
|
||||
|
||||
public static get lfsDirectoryFull(): string {
|
||||
return path.join(CloudRunnerState.repoPathFull, `.git`, `lfs`);
|
||||
}
|
||||
|
||||
public static get purgeRemoteCaching(): boolean {
|
||||
return process.env.PURGE_REMOTE_BUILDER_CACHE !== undefined;
|
||||
}
|
||||
|
||||
public static get lfsCacheFolderFull() {
|
||||
return path.join(CloudRunnerState.cacheFolderFull, `lfs`);
|
||||
}
|
||||
|
||||
public static get libraryCacheFolderFull() {
|
||||
return path.join(CloudRunnerState.cacheFolderFull, `Library`);
|
||||
}
|
||||
|
||||
public static get unityBuilderRepoUrl(): string {
|
||||
return `https://${CloudRunnerState.buildParams.githubToken}@github.com/game-ci/unity-builder.git`;
|
||||
}
|
||||
|
||||
public static get targetBuildRepoUrl(): string {
|
||||
return `https://${CloudRunnerState.buildParams.githubToken}@github.com/${CloudRunnerState.buildParams.githubRepo}.git`;
|
||||
}
|
||||
|
||||
public static get buildVolumeFolder() {
|
||||
return 'data';
|
||||
}
|
||||
|
||||
public static get cacheFolder() {
|
||||
return 'cache';
|
||||
}
|
||||
}
|
||||
@ -1,77 +0,0 @@
|
||||
import path from 'path';
|
||||
import { Input } from '../..';
|
||||
import { CloudRunnerBuildCommandProcessor } from '../services/cloud-runner-build-command-process';
|
||||
import CloudRunnerEnvironmentVariable from '../services/cloud-runner-environment-variable';
|
||||
import CloudRunnerLogger from '../services/cloud-runner-logger';
|
||||
import CloudRunnerSecret from '../services/cloud-runner-secret';
|
||||
import { CloudRunnerState } from '../state/cloud-runner-state';
|
||||
import { CloudRunnerStepState } from '../state/cloud-runner-step-state';
|
||||
import { StepInterface } from './step-interface';
|
||||
|
||||
export class BuildStep implements StepInterface {
|
||||
async run(cloudRunnerStepState: CloudRunnerStepState) {
|
||||
return await BuildStep.BuildStep(
|
||||
cloudRunnerStepState.image,
|
||||
cloudRunnerStepState.environment,
|
||||
cloudRunnerStepState.secrets,
|
||||
);
|
||||
}
|
||||
|
||||
private static async BuildStep(
|
||||
image: string,
|
||||
environmentVariables: CloudRunnerEnvironmentVariable[],
|
||||
secrets: CloudRunnerSecret[],
|
||||
) {
|
||||
CloudRunnerLogger.logLine(` `);
|
||||
CloudRunnerLogger.logLine('Starting part 2/2 (build unity project)');
|
||||
const hooks = CloudRunnerBuildCommandProcessor.getHooks().filter((x) => x.step.includes(`setup`));
|
||||
return await CloudRunnerState.CloudRunnerProviderPlatform.runTask(
|
||||
CloudRunnerState.buildParams.buildGuid,
|
||||
image,
|
||||
`${hooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
|
||||
export GITHUB_WORKSPACE="${CloudRunnerState.repoPathFull}"
|
||||
cp -r "${path
|
||||
.join(CloudRunnerState.builderPathFull, 'dist', 'default-build-script')
|
||||
.replace(/\\/g, `/`)}" "/UnityBuilderAction"
|
||||
cp -r "${path
|
||||
.join(CloudRunnerState.builderPathFull, 'dist', 'platforms', 'ubuntu', 'entrypoint.sh')
|
||||
.replace(/\\/g, `/`)}" "/entrypoint.sh"
|
||||
cp -r "${path
|
||||
.join(CloudRunnerState.builderPathFull, 'dist', 'platforms', 'ubuntu', 'steps')
|
||||
.replace(/\\/g, `/`)}" "/steps"
|
||||
chmod -R +x "/entrypoint.sh"
|
||||
chmod -R +x "/steps"
|
||||
/entrypoint.sh
|
||||
apt-get update
|
||||
apt-get install -y -q zip tree
|
||||
cd "${CloudRunnerState.libraryFolderFull.replace(/\\/g, `/`)}/.."
|
||||
zip -r "lib-${CloudRunnerState.buildParams.buildGuid}.zip" "Library"
|
||||
mv "lib-${CloudRunnerState.buildParams.buildGuid}.zip" "${CloudRunnerState.cacheFolderFull.replace(
|
||||
/\\/g,
|
||||
`/`,
|
||||
)}/Library"
|
||||
cd "${CloudRunnerState.repoPathFull.replace(/\\/g, `/`)}"
|
||||
${Input.cloudRunnerTests ? '' : '#'} tree -lh
|
||||
zip -r "build-${CloudRunnerState.buildParams.buildGuid}.zip" "build"
|
||||
${Input.cloudRunnerTests ? '' : '#'} tree -lh
|
||||
${Input.cloudRunnerTests ? '' : '#'} tree -lh "${CloudRunnerState.cacheFolderFull.replace(/\\/g, `/`)}"
|
||||
mv "build-${CloudRunnerState.buildParams.buildGuid}.zip" "${CloudRunnerState.cacheFolderFull.replace(
|
||||
/\\/g,
|
||||
`/`,
|
||||
)}"
|
||||
chmod +x ${path.join(CloudRunnerState.builderPathFull, 'dist', `index.js`).replace(/\\/g, `/`)}
|
||||
node ${path
|
||||
.join(CloudRunnerState.builderPathFull, 'dist', `index.js`)
|
||||
.replace(/\\/g, `/`)} -m cache-push "Library" "lib-${
|
||||
CloudRunnerState.buildParams.buildGuid
|
||||
}.zip" "${CloudRunnerState.cacheFolderFull.replace(/\\/g, `/`)}/Library"
|
||||
${Input.cloudRunnerTests ? '' : '#'} tree -lh "${CloudRunnerState.cacheFolderFull}"
|
||||
${hooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
|
||||
`,
|
||||
`/${CloudRunnerState.buildVolumeFolder}`,
|
||||
`/${CloudRunnerState.projectPathFull}`,
|
||||
environmentVariables,
|
||||
secrets,
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -1,65 +0,0 @@
|
||||
import path from 'path';
|
||||
import { Input } from '../..';
|
||||
import { CloudRunnerBuildCommandProcessor } from '../services/cloud-runner-build-command-process';
|
||||
import CloudRunnerEnvironmentVariable from '../services/cloud-runner-environment-variable';
|
||||
import CloudRunnerLogger from '../services/cloud-runner-logger';
|
||||
import CloudRunnerSecret from '../services/cloud-runner-secret';
|
||||
import { CloudRunnerState } from '../state/cloud-runner-state';
|
||||
import { CloudRunnerStepState } from '../state/cloud-runner-step-state';
|
||||
import { StepInterface } from './step-interface';
|
||||
|
||||
export class SetupStep implements StepInterface {
|
||||
async run(cloudRunnerStepState: CloudRunnerStepState) {
|
||||
try {
|
||||
return await SetupStep.downloadRepository(
|
||||
cloudRunnerStepState.image,
|
||||
cloudRunnerStepState.environment,
|
||||
cloudRunnerStepState.secrets,
|
||||
);
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private static getCloudRunnerBranch() {
|
||||
return process.env.CLOUD_RUNNER_BRANCH?.includes('/')
|
||||
? process.env.CLOUD_RUNNER_BRANCH.split('/').reverse()[0]
|
||||
: process.env.CLOUD_RUNNER_BRANCH;
|
||||
}
|
||||
|
||||
private static async downloadRepository(
|
||||
image: string,
|
||||
environmentVariables: CloudRunnerEnvironmentVariable[],
|
||||
secrets: CloudRunnerSecret[],
|
||||
) {
|
||||
try {
|
||||
CloudRunnerLogger.log(` `);
|
||||
CloudRunnerLogger.logLine('Starting step 1/2 (setup game files from repository)');
|
||||
const hooks = CloudRunnerBuildCommandProcessor.getHooks().filter((x) => x.step.includes(`setup`));
|
||||
return await CloudRunnerState.CloudRunnerProviderPlatform.runTask(
|
||||
CloudRunnerState.buildParams.buildGuid,
|
||||
image,
|
||||
`apk update -q
|
||||
apk add git-lfs jq tree zip unzip nodejs -q
|
||||
${hooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
|
||||
export GIT_DISCOVERY_ACROSS_FILESYSTEM=1
|
||||
mkdir -p ${CloudRunnerState.builderPathFull.replace(/\\/g, `/`)}
|
||||
git clone -q -b ${SetupStep.getCloudRunnerBranch()} ${
|
||||
CloudRunnerState.unityBuilderRepoUrl
|
||||
} "${CloudRunnerState.builderPathFull.replace(/\\/g, `/`)}"
|
||||
${Input.cloudRunnerTests ? '' : '#'} tree ${CloudRunnerState.builderPathFull.replace(/\\/g, `/`)}
|
||||
chmod +x ${path.join(CloudRunnerState.builderPathFull, 'dist', `index.js`).replace(/\\/g, `/`)}
|
||||
node ${path.join(CloudRunnerState.builderPathFull, 'dist', `index.js`).replace(/\\/g, `/`)} -m remote-cli
|
||||
${hooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
|
||||
`,
|
||||
`/${CloudRunnerState.buildVolumeFolder}`,
|
||||
`/${CloudRunnerState.buildVolumeFolder}/`,
|
||||
environmentVariables,
|
||||
secrets,
|
||||
);
|
||||
} catch (error) {
|
||||
CloudRunnerLogger.logLine(`Failed download repository step 1/2`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,8 +0,0 @@
|
||||
import { CloudRunnerStepState } from '../state/cloud-runner-step-state';
|
||||
|
||||
export interface StepInterface {
|
||||
run(
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
cloudRunnerStepState: CloudRunnerStepState,
|
||||
);
|
||||
}
|
||||
@ -1,12 +1,12 @@
|
||||
import CloudRunnerLogger from '../services/cloud-runner-logger';
|
||||
import { TaskParameterSerializer } from '../services/task-parameter-serializer';
|
||||
import { CloudRunnerState } from '../state/cloud-runner-state';
|
||||
import { CloudRunnerStepState } from '../state/cloud-runner-step-state';
|
||||
import { BuildStep } from '../steps/build-step';
|
||||
import { SetupStep } from '../steps/setup-step';
|
||||
import { CloudRunnerFolders } from '../services/cloud-runner-folders';
|
||||
import { CloudRunnerStepState } from '../cloud-runner-step-state';
|
||||
import { CustomWorkflow } from './custom-workflow';
|
||||
import { WorkflowInterface } from './workflow-interface';
|
||||
import * as core from '@actions/core';
|
||||
import { CloudRunnerBuildCommandProcessor } from '../services/cloud-runner-build-command-process';
|
||||
import path from 'path';
|
||||
import CloudRunner from '../cloud-runner';
|
||||
|
||||
export class BuildAutomationWorkflow implements WorkflowInterface {
|
||||
async run(cloudRunnerStepState: CloudRunnerStepState) {
|
||||
@ -21,41 +21,36 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
|
||||
try {
|
||||
CloudRunnerLogger.log(`Cloud Runner is running standard build automation`);
|
||||
|
||||
core.startGroup('pre build steps');
|
||||
if (!CloudRunner.buildParameters.isCliMode) core.startGroup('pre build steps');
|
||||
let output = '';
|
||||
if (CloudRunnerState.buildParams.preBuildSteps !== '') {
|
||||
output += await CustomWorkflow.runCustomJob(CloudRunnerState.buildParams.preBuildSteps);
|
||||
if (CloudRunner.buildParameters.preBuildSteps !== '') {
|
||||
output += await CustomWorkflow.runCustomJob(CloudRunner.buildParameters.preBuildSteps);
|
||||
}
|
||||
core.endGroup();
|
||||
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
|
||||
CloudRunnerLogger.logWithTime('Configurable pre build step(s) time');
|
||||
|
||||
core.startGroup('setup');
|
||||
output += await new SetupStep().run(
|
||||
new CloudRunnerStepState(
|
||||
'alpine/git',
|
||||
TaskParameterSerializer.readBuildEnvironmentVariables(),
|
||||
CloudRunnerState.defaultSecrets,
|
||||
),
|
||||
);
|
||||
core.endGroup();
|
||||
CloudRunnerLogger.logWithTime('Download repository step time');
|
||||
if (!CloudRunner.buildParameters.isCliMode) core.startGroup('build');
|
||||
CloudRunnerLogger.log(baseImage.toString());
|
||||
CloudRunnerLogger.logLine(` `);
|
||||
CloudRunnerLogger.logLine('Starting build automation job');
|
||||
|
||||
core.startGroup('build');
|
||||
output += await new BuildStep().run(
|
||||
new CloudRunnerStepState(
|
||||
baseImage,
|
||||
TaskParameterSerializer.readBuildEnvironmentVariables(),
|
||||
CloudRunnerState.defaultSecrets,
|
||||
),
|
||||
output += await CloudRunner.Provider.runTask(
|
||||
CloudRunner.buildParameters.buildGuid,
|
||||
baseImage.toString(),
|
||||
BuildAutomationWorkflow.BuildWorkflow,
|
||||
`/${CloudRunnerFolders.buildVolumeFolder}`,
|
||||
`/${CloudRunnerFolders.buildVolumeFolder}/`,
|
||||
CloudRunner.cloudRunnerEnvironmentVariables,
|
||||
CloudRunner.defaultSecrets,
|
||||
);
|
||||
core.endGroup();
|
||||
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
|
||||
CloudRunnerLogger.logWithTime('Build time');
|
||||
|
||||
core.startGroup('post build steps');
|
||||
if (CloudRunnerState.buildParams.postBuildSteps !== '') {
|
||||
output += await CustomWorkflow.runCustomJob(CloudRunnerState.buildParams.postBuildSteps);
|
||||
if (!CloudRunner.buildParameters.isCliMode) core.startGroup('post build steps');
|
||||
if (CloudRunner.buildParameters.postBuildSteps !== '') {
|
||||
output += await CustomWorkflow.runCustomJob(CloudRunner.buildParameters.postBuildSteps);
|
||||
}
|
||||
core.endGroup();
|
||||
if (!CloudRunner.buildParameters.isCliMode) core.endGroup();
|
||||
CloudRunnerLogger.logWithTime('Configurable post build step(s) time');
|
||||
|
||||
CloudRunnerLogger.log(`Cloud Runner finished running standard build automation`);
|
||||
@ -65,4 +60,62 @@ export class BuildAutomationWorkflow implements WorkflowInterface {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private static get BuildWorkflow() {
|
||||
const setupHooks = CloudRunnerBuildCommandProcessor.getHooks(CloudRunner.buildParameters.customJobHooks).filter(
|
||||
(x) => x.step.includes(`setup`),
|
||||
);
|
||||
const buildHooks = CloudRunnerBuildCommandProcessor.getHooks(CloudRunner.buildParameters.customJobHooks).filter(
|
||||
(x) => x.step.includes(`build`),
|
||||
);
|
||||
const builderPath = path.join(CloudRunnerFolders.builderPathAbsolute, 'dist', `index.js`).replace(/\\/g, `/`);
|
||||
return `apt-get update > /dev/null
|
||||
apt-get install -y zip tree npm git-lfs jq unzip git > /dev/null
|
||||
npm install -g n > /dev/null
|
||||
n stable > /dev/null
|
||||
${setupHooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
|
||||
export GITHUB_WORKSPACE="${CloudRunnerFolders.repoPathAbsolute.replace(/\\/g, `/`)}"
|
||||
${BuildAutomationWorkflow.setupCommands(builderPath)}
|
||||
${setupHooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}
|
||||
${buildHooks.filter((x) => x.hook.includes(`before`)).map((x) => x.commands) || ' '}
|
||||
${BuildAutomationWorkflow.BuildCommands(builderPath, CloudRunner.buildParameters.buildGuid)}
|
||||
${buildHooks.filter((x) => x.hook.includes(`after`)).map((x) => x.commands) || ' '}`;
|
||||
}
|
||||
|
||||
private static setupCommands(builderPath) {
|
||||
return `export GIT_DISCOVERY_ACROSS_FILESYSTEM=1
|
||||
echo "game ci cloud runner clone"
|
||||
mkdir -p ${CloudRunnerFolders.builderPathAbsolute.replace(/\\/g, `/`)}
|
||||
git clone -q -b ${CloudRunner.buildParameters.cloudRunnerBranch} ${
|
||||
CloudRunnerFolders.unityBuilderRepoUrl
|
||||
} "${CloudRunnerFolders.builderPathAbsolute.replace(/\\/g, `/`)}"
|
||||
chmod +x ${builderPath}
|
||||
echo "game ci cloud runner bootstrap"
|
||||
node ${builderPath} -m remote-cli`;
|
||||
}
|
||||
|
||||
private static BuildCommands(builderPath, guid) {
|
||||
const linuxCacheFolder = CloudRunnerFolders.cacheFolderFull.replace(/\\/g, `/`);
|
||||
const distFolder = path.join(CloudRunnerFolders.builderPathAbsolute, 'dist');
|
||||
const ubuntuPlatformsFolder = path.join(CloudRunnerFolders.builderPathAbsolute, 'dist', 'platforms', 'ubuntu');
|
||||
return `echo "game ci cloud runner init"
|
||||
mkdir -p ${`${CloudRunnerFolders.projectBuildFolderAbsolute}/build`.replace(/\\/g, `/`)}
|
||||
cd ${CloudRunnerFolders.projectPathAbsolute}
|
||||
cp -r "${path.join(distFolder, 'default-build-script').replace(/\\/g, `/`)}" "/UnityBuilderAction"
|
||||
cp -r "${path.join(ubuntuPlatformsFolder, 'entrypoint.sh').replace(/\\/g, `/`)}" "/entrypoint.sh"
|
||||
cp -r "${path.join(ubuntuPlatformsFolder, 'steps').replace(/\\/g, `/`)}" "/steps"
|
||||
chmod -R +x "/entrypoint.sh"
|
||||
chmod -R +x "/steps"
|
||||
echo "game ci cloud runner start"
|
||||
/entrypoint.sh
|
||||
echo "game ci cloud runner push library to cache"
|
||||
chmod +x ${builderPath}
|
||||
node ${builderPath} -m cache-push --cachePushFrom ${
|
||||
CloudRunnerFolders.libraryFolderAbsolute
|
||||
} --artifactName lib-${guid} --cachePushTo ${linuxCacheFolder}/Library
|
||||
echo "game ci cloud runner push build to cache"
|
||||
node ${builderPath} -m cache-push --cachePushFrom ${
|
||||
CloudRunnerFolders.projectBuildFolderAbsolute
|
||||
} --artifactName build-${guid} --cachePushTo ${`${linuxCacheFolder}/build`.replace(/\\/g, `/`)}`;
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,44 +1,43 @@
|
||||
import CloudRunnerLogger from '../services/cloud-runner-logger';
|
||||
import CloudRunnerSecret from '../services/cloud-runner-secret';
|
||||
import { CloudRunnerState } from '../state/cloud-runner-state';
|
||||
import { CloudRunnerFolders } from '../services/cloud-runner-folders';
|
||||
import YAML from 'yaml';
|
||||
import { Input } from '../..';
|
||||
import { TaskParameterSerializer } from '../services/task-parameter-serializer';
|
||||
import { CloudRunner, Input } from '../..';
|
||||
|
||||
export class CustomWorkflow {
|
||||
public static async runCustomJob(buildSteps) {
|
||||
try {
|
||||
CloudRunnerLogger.log(`Cloud Runner is running in custom job mode`);
|
||||
if (Input.cloudRunnerTests) {
|
||||
if (CloudRunner.buildParameters.cloudRunnerIntegrationTests) {
|
||||
CloudRunnerLogger.log(`Parsing build steps: ${buildSteps}`);
|
||||
}
|
||||
try {
|
||||
buildSteps = YAML.parse(buildSteps);
|
||||
let output = '';
|
||||
for (const step of buildSteps) {
|
||||
const stepSecrets: CloudRunnerSecret[] = step.secrets.map((x) => {
|
||||
const secret: CloudRunnerSecret = {
|
||||
ParameterKey: x.name,
|
||||
EnvironmentVariable: Input.ToEnvVarFormat(x.name),
|
||||
ParameterValue: x.value,
|
||||
};
|
||||
return secret;
|
||||
});
|
||||
output += await CloudRunnerState.CloudRunnerProviderPlatform.runTask(
|
||||
CloudRunnerState.buildParams.buildGuid,
|
||||
step['image'],
|
||||
step['commands'],
|
||||
`/${CloudRunnerState.buildVolumeFolder}`,
|
||||
`/${CloudRunnerState.buildVolumeFolder}/`,
|
||||
TaskParameterSerializer.readBuildEnvironmentVariables(),
|
||||
[...CloudRunnerState.defaultSecrets, ...stepSecrets],
|
||||
);
|
||||
}
|
||||
return output;
|
||||
} catch (error) {
|
||||
CloudRunnerLogger.log(`failed to parse a custom job "${buildSteps}"`);
|
||||
throw error;
|
||||
}
|
||||
let output = '';
|
||||
for (const step of buildSteps) {
|
||||
const stepSecrets: CloudRunnerSecret[] = step.secrets.map((x) => {
|
||||
const secret: CloudRunnerSecret = {
|
||||
ParameterKey: x.name,
|
||||
EnvironmentVariable: Input.ToEnvVarFormat(x.name),
|
||||
ParameterValue: x.value,
|
||||
};
|
||||
return secret;
|
||||
});
|
||||
output += await CloudRunner.Provider.runTask(
|
||||
CloudRunner.buildParameters.buildGuid,
|
||||
step['image'],
|
||||
step['commands'],
|
||||
`/${CloudRunnerFolders.buildVolumeFolder}`,
|
||||
`/${CloudRunnerFolders.buildVolumeFolder}/`,
|
||||
CloudRunner.cloudRunnerEnvironmentVariables,
|
||||
[...CloudRunner.defaultSecrets, ...stepSecrets],
|
||||
);
|
||||
}
|
||||
return output;
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
@ -1,10 +1,8 @@
|
||||
import { CloudRunnerState } from '../state/cloud-runner-state';
|
||||
import { CloudRunnerStepState } from '../state/cloud-runner-step-state';
|
||||
import { CloudRunnerStepState } from '../cloud-runner-step-state';
|
||||
import { CustomWorkflow } from './custom-workflow';
|
||||
import { WorkflowInterface } from './workflow-interface';
|
||||
import { BuildAutomationWorkflow } from './build-automation-workflow';
|
||||
import { TaskParameterSerializer } from '../services/task-parameter-serializer';
|
||||
import { SetupStep } from '../steps/setup-step';
|
||||
import CloudRunner from '../cloud-runner';
|
||||
|
||||
export class WorkflowCompositionRoot implements WorkflowInterface {
|
||||
async run(cloudRunnerStepState: CloudRunnerStepState) {
|
||||
@ -17,23 +15,11 @@ export class WorkflowCompositionRoot implements WorkflowInterface {
|
||||
|
||||
private static async runJob(baseImage: any) {
|
||||
try {
|
||||
if (CloudRunnerState.buildParams.customJob === `setup`) {
|
||||
return await new SetupStep().run(
|
||||
new CloudRunnerStepState(
|
||||
baseImage,
|
||||
TaskParameterSerializer.readBuildEnvironmentVariables(),
|
||||
CloudRunnerState.defaultSecrets,
|
||||
),
|
||||
);
|
||||
} else if (CloudRunnerState.buildParams.customJob !== '') {
|
||||
return await CustomWorkflow.runCustomJob(CloudRunnerState.buildParams.customJob);
|
||||
if (CloudRunner.buildParameters.customJob !== '') {
|
||||
return await CustomWorkflow.runCustomJob(CloudRunner.buildParameters.customJob);
|
||||
}
|
||||
return await new BuildAutomationWorkflow().run(
|
||||
new CloudRunnerStepState(
|
||||
baseImage,
|
||||
TaskParameterSerializer.readBuildEnvironmentVariables(),
|
||||
CloudRunnerState.defaultSecrets,
|
||||
),
|
||||
new CloudRunnerStepState(baseImage, CloudRunner.cloudRunnerEnvironmentVariables, CloudRunner.defaultSecrets),
|
||||
);
|
||||
} catch (error) {
|
||||
throw error;
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { CloudRunnerStepState } from '../state/cloud-runner-step-state';
|
||||
import { CloudRunnerStepState } from '../cloud-runner-step-state';
|
||||
|
||||
export interface WorkflowInterface {
|
||||
run(
|
||||
|
||||
@ -1,9 +1,11 @@
|
||||
import Platform from './platform';
|
||||
|
||||
import BuildParameters from './build-parameters';
|
||||
|
||||
class ImageTag {
|
||||
public repository: string;
|
||||
public name: string;
|
||||
public cloudRunnerBuilderPlatform!: string | undefined;
|
||||
public editorVersion: string;
|
||||
public targetPlatform: any;
|
||||
public builderPlatform: string;
|
||||
@ -12,7 +14,7 @@ class ImageTag {
|
||||
public imagePlatformPrefix: string;
|
||||
|
||||
constructor(imageProperties: Partial<BuildParameters>) {
|
||||
const { editorVersion = '2019.2.11f1', targetPlatform, customImage } = imageProperties;
|
||||
const { editorVersion = '2019.2.11f1', targetPlatform, customImage, cloudRunnerBuilderPlatform } = imageProperties;
|
||||
|
||||
if (!ImageTag.versionPattern.test(editorVersion)) {
|
||||
throw new Error(`Invalid version "${editorVersion}".`);
|
||||
@ -27,8 +29,12 @@ class ImageTag {
|
||||
this.name = 'editor';
|
||||
this.editorVersion = editorVersion;
|
||||
this.targetPlatform = targetPlatform;
|
||||
this.cloudRunnerBuilderPlatform = cloudRunnerBuilderPlatform;
|
||||
const isCloudRunnerLocal = cloudRunnerBuilderPlatform === 'local' || cloudRunnerBuilderPlatform === undefined;
|
||||
this.builderPlatform = ImageTag.getTargetPlatformToTargetPlatformSuffixMap(targetPlatform, editorVersion);
|
||||
this.imagePlatformPrefix = ImageTag.getImagePlatformPrefixes(process.platform);
|
||||
this.imagePlatformPrefix = ImageTag.getImagePlatformPrefixes(
|
||||
isCloudRunnerLocal ? process.platform : cloudRunnerBuilderPlatform,
|
||||
);
|
||||
this.imageRollingVersion = 1; // will automatically roll to the latest non-breaking version.
|
||||
}
|
||||
|
||||
@ -155,5 +161,4 @@ class ImageTag {
|
||||
return `${image}:${tag}`; // '0' here represents the docker repo version
|
||||
}
|
||||
}
|
||||
|
||||
export default ImageTag;
|
||||
|
||||
7
src/model/input-readers/generic-input-reader.ts
Normal file
7
src/model/input-readers/generic-input-reader.ts
Normal file
@ -0,0 +1,7 @@
|
||||
import { CloudRunnerSystem } from '../cloud-runner/services/cloud-runner-system';
|
||||
|
||||
export class GenericInputReader {
|
||||
public static async Run(command) {
|
||||
return await CloudRunnerSystem.Run(command, false, true);
|
||||
}
|
||||
}
|
||||
@ -1,22 +1,22 @@
|
||||
import { assert } from 'console';
|
||||
import System from '../system';
|
||||
import fs from 'fs';
|
||||
import { CloudRunnerSystem } from '../cli/remote-client/remote-client-services/cloud-runner-system';
|
||||
import { CloudRunnerSystem } from '../cloud-runner/services/cloud-runner-system';
|
||||
import CloudRunnerLogger from '../cloud-runner/services/cloud-runner-logger';
|
||||
|
||||
export class GitRepoReader {
|
||||
static GetSha() {
|
||||
return '';
|
||||
}
|
||||
|
||||
public static async GetRemote() {
|
||||
return (await CloudRunnerSystem.Run(`git remote -v`))
|
||||
.split(' ')[1]
|
||||
.split('https://github.com/')[1]
|
||||
.split('.git')[0];
|
||||
assert(fs.existsSync(`.git`));
|
||||
const value = (await CloudRunnerSystem.Run(`git remote -v`, false, true)).replace(/ /g, ``);
|
||||
CloudRunnerLogger.log(`value ${value}`);
|
||||
assert(value.includes('github.com'));
|
||||
return value.split('github.com/')[1].split('.git')[0];
|
||||
}
|
||||
|
||||
public static async GetBranch() {
|
||||
assert(fs.existsSync(`.git`));
|
||||
return (await System.run(`git branch`, [], {}, false)).split('*')[1].split(`\n`)[0].replace(/ /g, ``);
|
||||
return (await CloudRunnerSystem.Run(`git branch --show-current`, false, true))
|
||||
.split('\n')[0]
|
||||
.replace(/ /g, ``)
|
||||
.replace('/head', '');
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,14 +1,14 @@
|
||||
import { CloudRunnerSystem } from '../cli/remote-client/remote-client-services/cloud-runner-system';
|
||||
import { CloudRunnerSystem } from '../cloud-runner/services/cloud-runner-system';
|
||||
import * as core from '@actions/core';
|
||||
|
||||
export class GithubCliReader {
|
||||
static async GetGitHubAuthToken() {
|
||||
try {
|
||||
const authStatus = await CloudRunnerSystem.Run(`gh auth status`, true);
|
||||
const authStatus = await CloudRunnerSystem.Run(`gh auth status`, true, true);
|
||||
if (authStatus.includes('You are not logged') || authStatus === '') {
|
||||
return '';
|
||||
}
|
||||
return (await CloudRunnerSystem.Run(`gh auth status -t`))
|
||||
return (await CloudRunnerSystem.Run(`gh auth status -t`, false, true))
|
||||
.split(`Token: `)[1]
|
||||
.replace(/ /g, '')
|
||||
.replace(/\n/g, '');
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { GitRepoReader } from './input-readers/git-repo';
|
||||
import { GithubCliReader } from './input-readers/github-cli';
|
||||
import { Cli } from './cli/cli';
|
||||
import CloudRunnerQueryOverride from './cloud-runner/services/cloud-runner-query-override';
|
||||
import Platform from './platform';
|
||||
|
||||
const core = require('@actions/core');
|
||||
@ -14,64 +14,69 @@ const core = require('@actions/core');
|
||||
* Todo: rename to UserInput and remove anything that is not direct input from the user / ci workflow
|
||||
*/
|
||||
class Input {
|
||||
public static cliOptions;
|
||||
public static githubInputEnabled: boolean = true;
|
||||
|
||||
// also enabled debug logging for cloud runner
|
||||
static get cloudRunnerTests(): boolean {
|
||||
return Input.getInput(`cloudRunnerTests`) || Input.getInput(`CloudRunnerTests`) || false;
|
||||
}
|
||||
public static getInput(query) {
|
||||
if (Input.githubInputEnabled) {
|
||||
const coreInput = core.getInput(query);
|
||||
if (coreInput && coreInput !== '') {
|
||||
return coreInput;
|
||||
}
|
||||
}
|
||||
const alternativeQuery = Input.ToEnvVarFormat(query);
|
||||
|
||||
private static getInput(query) {
|
||||
const coreInput = core.getInput(query);
|
||||
if (Input.githubInputEnabled && coreInput && coreInput !== '') {
|
||||
return coreInput;
|
||||
// query input sources
|
||||
if (Cli.query(query, alternativeQuery)) {
|
||||
return Cli.query(query, alternativeQuery);
|
||||
}
|
||||
|
||||
return Input.cliOptions !== undefined && Input.cliOptions[query] !== undefined
|
||||
? Input.cliOptions[query]
|
||||
: process.env[query] !== undefined
|
||||
? process.env[query]
|
||||
: process.env[Input.ToEnvVarFormat(query)]
|
||||
? process.env[Input.ToEnvVarFormat(query)]
|
||||
: '';
|
||||
if (CloudRunnerQueryOverride.query(query, alternativeQuery)) {
|
||||
return CloudRunnerQueryOverride.query(query, alternativeQuery);
|
||||
}
|
||||
|
||||
if (process.env[query] !== undefined) {
|
||||
return process.env[query];
|
||||
}
|
||||
|
||||
if (alternativeQuery !== query && process.env[alternativeQuery] !== undefined) {
|
||||
return process.env[alternativeQuery];
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
static get region(): string {
|
||||
return Input.getInput('region') || 'eu-west-2';
|
||||
}
|
||||
|
||||
static async githubRepo() {
|
||||
return (
|
||||
Input.getInput('GITHUB_REPOSITORY') ||
|
||||
Input.getInput('GITHUB_REPO') ||
|
||||
// todo - move this to some class specific for determining additional information
|
||||
(await GitRepoReader.GetRemote()) ||
|
||||
'game-ci/unity-builder'
|
||||
);
|
||||
static get githubRepo() {
|
||||
return Input.getInput('GITHUB_REPOSITORY') || Input.getInput('GITHUB_REPO') || undefined;
|
||||
}
|
||||
|
||||
static async branch() {
|
||||
if (await GitRepoReader.GetBranch()) {
|
||||
// todo - move this to some class specific for determining additional information
|
||||
return await GitRepoReader.GetBranch();
|
||||
} else if (Input.getInput(`GITHUB_REF`)) {
|
||||
return Input.getInput(`GITHUB_REF`).replace('refs/', '').replace(`head/`, '');
|
||||
static get branch() {
|
||||
if (Input.getInput(`GITHUB_REF`)) {
|
||||
return Input.getInput(`GITHUB_REF`).replace('refs/', '').replace(`head/`, '').replace(`heads/`, '');
|
||||
} else if (Input.getInput('branch')) {
|
||||
return Input.getInput('branch');
|
||||
} else {
|
||||
return 'main';
|
||||
return '';
|
||||
}
|
||||
}
|
||||
static get cloudRunnerBuilderPlatform() {
|
||||
const input = Input.getInput('cloudRunnerBuilderPlatform');
|
||||
if (input) {
|
||||
return input;
|
||||
}
|
||||
if (Input.cloudRunnerCluster !== 'local') {
|
||||
return 'linux';
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
static get gitSha() {
|
||||
if (Input.getInput(`GITHUB_SHA`)) {
|
||||
return Input.getInput(`GITHUB_SHA`);
|
||||
} else if (Input.getInput(`GitSHA`)) {
|
||||
return Input.getInput(`GitSHA`);
|
||||
} else if (GitRepoReader.GetSha()) {
|
||||
// todo - move this to some class specific for determining additional information
|
||||
return GitRepoReader.GetSha();
|
||||
}
|
||||
}
|
||||
|
||||
@ -88,7 +93,7 @@ class Input {
|
||||
}
|
||||
|
||||
static get customImage() {
|
||||
return Input.getInput('customImage');
|
||||
return Input.getInput('customImage') || '';
|
||||
}
|
||||
|
||||
static get projectPath() {
|
||||
@ -165,8 +170,36 @@ class Input {
|
||||
return Input.getInput('sshAgent') || '';
|
||||
}
|
||||
|
||||
static async gitPrivateToken() {
|
||||
return Input.getInput('gitPrivateToken') || (await Input.githubToken());
|
||||
static get gitPrivateToken() {
|
||||
return core.getInput('gitPrivateToken') || false;
|
||||
}
|
||||
|
||||
static get customJob() {
|
||||
return Input.getInput('customJob') || '';
|
||||
}
|
||||
|
||||
static customJobHooks() {
|
||||
return Input.getInput('customJobHooks') || '';
|
||||
}
|
||||
|
||||
static cachePushOverrideCommand() {
|
||||
return Input.getInput('cachePushOverrideCommand') || '';
|
||||
}
|
||||
|
||||
static cachePullOverrideCommand() {
|
||||
return Input.getInput('cachePullOverrideCommand') || '';
|
||||
}
|
||||
|
||||
static readInputFromOverrideList() {
|
||||
return Input.getInput('readInputFromOverrideList') || '';
|
||||
}
|
||||
|
||||
static readInputOverrideCommand() {
|
||||
return Input.getInput('readInputOverrideCommand') || '';
|
||||
}
|
||||
|
||||
static get cloudRunnerBranch() {
|
||||
return Input.getInput('cloudRunnerBranch') || 'cloud-runner-develop';
|
||||
}
|
||||
|
||||
static get chownFilesTo() {
|
||||
@ -187,15 +220,14 @@ class Input {
|
||||
return Input.getInput('preBuildSteps') || '';
|
||||
}
|
||||
|
||||
static get customJob() {
|
||||
return Input.getInput('customJob') || '';
|
||||
}
|
||||
|
||||
static get awsBaseStackName() {
|
||||
return Input.getInput('awsBaseStackName') || 'game-ci';
|
||||
}
|
||||
|
||||
static get cloudRunnerCluster() {
|
||||
if (Cli.isCliMode) {
|
||||
return Input.getInput('cloudRunnerCluster') || 'aws';
|
||||
}
|
||||
return Input.getInput('cloudRunnerCluster') || 'local';
|
||||
}
|
||||
|
||||
@ -207,11 +239,6 @@ class Input {
|
||||
return Input.getInput('cloudRunnerMemory') || '750M';
|
||||
}
|
||||
|
||||
static async githubToken() {
|
||||
// Todo - move GitHubCLI out of the simple input class. It is in fact not input from the user.
|
||||
return Input.getInput('githubToken') || (await GithubCliReader.GetGitHubAuthToken()) || '';
|
||||
}
|
||||
|
||||
static get kubeConfig() {
|
||||
return Input.getInput('kubeConfig') || '';
|
||||
}
|
||||
@ -224,7 +251,30 @@ class Input {
|
||||
return Input.getInput('kubeVolumeSize') || '5Gi';
|
||||
}
|
||||
|
||||
static get kubeStorageClass(): string {
|
||||
return Input.getInput('kubeStorageClass') || '';
|
||||
}
|
||||
|
||||
static get checkDependencyHealthOverride(): string {
|
||||
return Input.getInput('checkDependencyHealthOverride') || '';
|
||||
}
|
||||
|
||||
static get startDependenciesOverride(): string {
|
||||
return Input.getInput('startDependenciesOverride') || '';
|
||||
}
|
||||
|
||||
static get cacheKey(): string {
|
||||
return Input.getInput('cacheKey') || Input.branch;
|
||||
}
|
||||
|
||||
static get cloudRunnerTests(): boolean {
|
||||
return Input.getInput(`cloudRunnerTests`) || false;
|
||||
}
|
||||
|
||||
public static ToEnvVarFormat(input: string) {
|
||||
if (input.toUpperCase() === input) {
|
||||
return input;
|
||||
}
|
||||
return input
|
||||
.replace(/([A-Z])/g, ' $1')
|
||||
.trim()
|
||||
|
||||
12
yarn.lock
12
yarn.lock
@ -1157,7 +1157,7 @@
|
||||
dependencies:
|
||||
"@types/node" "*"
|
||||
|
||||
"@types/node@*", "@types/node@^17.0.21":
|
||||
"@types/node@*":
|
||||
version "17.0.21"
|
||||
resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.21.tgz#864b987c0c68d07b4345845c3e63b75edd143644"
|
||||
integrity sha512-DBZCJbhII3r90XbQxI8Y9IjjiiOGlZ0Hr32omXIZvwwZ7p4DMMXGrKXVyPfuoBOri9XNtL0UK69jYIBIsRX3QQ==
|
||||
@ -1167,6 +1167,11 @@
|
||||
resolved "https://registry.yarnpkg.com/@types/node/-/node-10.17.60.tgz#35f3d6213daed95da7f0f73e75bcc6980e90597b"
|
||||
integrity sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==
|
||||
|
||||
"@types/node@^17.0.23":
|
||||
version "17.0.23"
|
||||
resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.23.tgz#3b41a6e643589ac6442bdbd7a4a3ded62f33f7da"
|
||||
integrity sha512-UxDxWn7dl97rKVeVS61vErvw086aCYhDLyvRQZ5Rk65rZKepaFdm53GeqXaKBuOhED4e9uWq34IC3TdSdJJ2Gw==
|
||||
|
||||
"@types/normalize-package-data@^2.4.0":
|
||||
version "2.4.0"
|
||||
resolved "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz"
|
||||
@ -5366,6 +5371,11 @@ uuid@^3.3.2:
|
||||
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee"
|
||||
integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==
|
||||
|
||||
uuid@^8.3.2:
|
||||
version "8.3.2"
|
||||
resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2"
|
||||
integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==
|
||||
|
||||
v8-compile-cache@^2.0.3:
|
||||
version "2.3.0"
|
||||
resolved "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user