Remote builder refactoring (#264)

Remote builder refactoring
This commit is contained in:
Frostebite 2021-05-23 05:08:40 +01:00 committed by GitHub
parent 9fff362775
commit c96b8cf443
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 904 additions and 725 deletions

View File

@ -1,10 +1,10 @@
name: AWS
on:
push: { branches: [aws, aws-ts-clean] }
push: { branches: [aws, remote-builder/refactor] }
env:
AWS_REGION: "eu-west-1"
AWS_REGION: 'eu-west-1'
jobs:
buildForAllPlatforms:
@ -17,7 +17,7 @@ jobs:
projectPath:
- test-project
unityVersion:
# - 2019.2.11f1
# - 2019.2.11f1
- 2019.3.15f1
targetPlatform:
#- StandaloneOSX # Build a macOS standalone (Intel 64-bit).

View File

@ -51,36 +51,7 @@ Parameters:
EFSMountDirectory:
Type: String
Default: '/efsdata'
GithubToken:
Type: String
Default: '0'
UnityLicense:
Type: String
Default: '0'
UnityEmail:
Type: String
Default: '0'
UnityPassword:
Type: String
Default: '0'
UnitySerial:
Type: String
Default: '0'
AndroidKeystoreBase64:
Type: String
Default: '0'
AndroidKeystorePass:
Type: String
Default: '0'
AndroidKeyAliasPass:
Type: String
Default: '0'
AWSAccessKeyID:
Type: String
Default: '0'
AWSSecretAccessKey:
Type: String
Default: '0'
# template secrets p1 - input
Mappings:
SubnetConfig:
VPC:
@ -128,64 +99,8 @@ Resources:
'AWS::CloudFormation::Designer':
id: c6f18447-b879-4696-8873-f981b2cedd2b
GithubTokenSecret:
Type: AWS::SecretsManager::Secret
Properties:
Name: !Join [ "", [ 'GithubToken', !Ref BUILDID ] ]
SecretString: !Ref GithubToken
UnityLicenseSecret:
Type: AWS::SecretsManager::Secret
Properties:
Name: !Join [ "", [ 'UnityLicense', !Ref BUILDID ] ]
SecretString: !Ref UnityLicense
UnityEmailSecret:
Type: AWS::SecretsManager::Secret
Properties:
Name: !Join [ "", [ 'UnityEmail', !Ref BUILDID ] ]
SecretString: !Ref UnityEmail
UnityPasswordSecret:
Type: AWS::SecretsManager::Secret
Properties:
Name: !Join [ "", [ 'UnityPassword', !Ref BUILDID ] ]
SecretString: !Ref UnityPassword
UnitySerialSecret:
Type: AWS::SecretsManager::Secret
Properties:
Name: !Join [ "", [ 'UnitySerial', !Ref BUILDID ] ]
SecretString: !Ref UnitySerial
AndroidKeystoreBase64Secret:
Type: AWS::SecretsManager::Secret
Properties:
Name: !Join [ "", [ 'AndroidKeystoreBase64', !Ref BUILDID ] ]
SecretString: !Ref AndroidKeystoreBase64
AndroidKeystorePassSecret:
Type: AWS::SecretsManager::Secret
Properties:
Name: !Join [ "", [ 'AndroidKeystorePass', !Ref BUILDID ] ]
SecretString: !Ref AndroidKeystorePass
AndroidKeyAliasPassSecret:
Type: AWS::SecretsManager::Secret
Properties:
Name: !Join [ "", [ 'AndroidKeyAliasPass', !Ref BUILDID ] ]
SecretString: !Ref AndroidKeyAliasPass
AWSAccessKeyIDSecret:
Type: AWS::SecretsManager::Secret
Properties:
Name: !Join [ "", [ 'AWSAccessKeyID', !Ref BUILDID ] ]
SecretString: !Ref AWSAccessKeyID
AWSSecretAccessKeySecret:
Type: AWS::SecretsManager::Secret
Properties:
Name: !Join [ "", [ 'AWSSecretAccessKey', !Ref BUILDID ] ]
SecretString: !Ref AWSSecretAccessKey
# template secrets p2 - secret
TaskDefinition:
Type: 'AWS::ECS::TaskDefinition'
Properties:
@ -225,29 +140,13 @@ Resources:
Environment:
- Name: ALLOW_EMPTY_PASSWORD
Value: 'yes'
# template - env vars
MountPoints:
- SourceVolume: efs-data
ContainerPath: !Ref EFSMountDirectory
ReadOnly: false
Secrets:
- Name: 'GITHUB_TOKEN'
ValueFrom: !Ref GithubTokenSecret
- Name: 'UNITY_LICENSE'
ValueFrom: !Ref UnityLicenseSecret
- Name: 'UNITY_EMAIL'
ValueFrom: !Ref UnityEmailSecret
- Name: 'UNITY_PASSWORD'
ValueFrom: !Ref UnityPasswordSecret
- Name: 'UNITY_SERIAL'
ValueFrom: !Ref UnitySerialSecret
- Name: 'ANDROID_KEYSTORE_BASE64'
ValueFrom: !Ref AndroidKeystoreBase64Secret
- Name: 'ANDROID_KEYSTORE_PASS'
ValueFrom: !Ref AndroidKeystorePassSecret
- Name: 'AWS_ACCESS_KEY_ID'
ValueFrom: !Ref AWSAccessKeyIDSecret
- Name: 'AWS_SECRET_ACCESS_KEY'
ValueFrom: !Ref AWSSecretAccessKeySecret
# template secrets p3 - container def
LogConfiguration:
LogDriver: awslogs
Options:

BIN
dist/index.js generated vendored

Binary file not shown.

BIN
dist/index.js.map generated vendored

Binary file not shown.

View File

@ -1,5 +1,5 @@
import * as core from '@actions/core';
import { Action, BuildParameters, Cache, Docker, ImageTag, Kubernetes, Output, AWS } from './model';
import { Action, BuildParameters, Cache, Docker, ImageTag, Kubernetes, Output, RemoteBuilder } from './model';
async function run() {
try {
@ -20,7 +20,7 @@ async function run() {
case 'aws':
core.info('Building with AWS');
await AWS.runBuildJob(buildParameters, baseImage);
await RemoteBuilder.build(buildParameters, baseImage);
break;
// default and local case

View File

@ -1,611 +0,0 @@
import * as SDK from 'aws-sdk';
import { customAlphabet } from 'nanoid';
import * as fs from 'fs';
import * as core from '@actions/core';
import * as zlib from 'zlib';
const alphabet = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz';
const repositoryDirectoryName = 'repo';
const efsDirectoryName = 'data';
const cacheDirectoryName = 'cache';
class AWS {
static async runBuildJob(buildParameters, baseImage) {
try {
const nanoid = customAlphabet(alphabet, 4);
const buildUid = `${process.env.GITHUB_RUN_NUMBER}-${buildParameters.platform
.replace('Standalone', '')
.replace('standalone', '')}-${nanoid()}`;
const branchName = process.env.GITHUB_REF?.split('/').reverse()[0];
core.info('Starting part 1/4 (clone from github and restore cache)');
await this.run(
buildUid,
buildParameters.awsStackName,
'alpine/git',
['/bin/sh'],
[
'-c',
`apk update;
apk add unzip;
apk add git-lfs;
apk add jq;
# Get source repo for project to be built and game-ci repo for utilties
git clone https://${buildParameters.githubToken}@github.com/${process.env.GITHUB_REPOSITORY}.git ${buildUid}/${repositoryDirectoryName} -q
git clone https://${buildParameters.githubToken}@github.com/game-ci/unity-builder.git ${buildUid}/builder -q
cd /${efsDirectoryName}/${buildUid}/${repositoryDirectoryName}/
git checkout $GITHUB_SHA
cd /${efsDirectoryName}/
# Look for usable cache
if [ ! -d ${cacheDirectoryName} ]; then
mkdir ${cacheDirectoryName}
fi
cd ${cacheDirectoryName}
if [ ! -d "${branchName}" ]; then
mkdir "${branchName}"
fi
cd "${branchName}"
echo " "
echo "Cached Libraries for ${branchName} from previous builds:"
ls
echo " "
libDir="/${efsDirectoryName}/${buildUid}/${repositoryDirectoryName}/${buildParameters.projectPath}/Library"
if [ -d "$libDir" ]; then
rm -r "$libDir"
echo "Setup .gitignore to ignore Library folder and remove it from builds"
fi
echo 'Checking cache'
# Restore cache
latest=$(ls -t | head -1)
if [ ! -z "$latest" ]; then
echo "Library cache exists from build $latest from ${branchName}"
echo 'Creating empty Library folder for cache'
mkdir "$libDir"
unzip -q $latest -d '/${efsDirectoryName}/${buildUid}/${repositoryDirectoryName}/${buildParameters.projectPath}/Library/.'
else
echo 'Cache does not exist'
fi
# Print out important directories
echo ' '
echo 'Repo:'
ls /${efsDirectoryName}/${buildUid}/${repositoryDirectoryName}/
echo ' '
echo 'Project:'
ls /${efsDirectoryName}/${buildUid}/${repositoryDirectoryName}/${buildParameters.projectPath}
echo ' '
echo 'Library:'
ls /${efsDirectoryName}/${buildUid}/${repositoryDirectoryName}/${buildParameters.projectPath}/Library/
echo ' '
`,
],
`/${efsDirectoryName}`,
`/${efsDirectoryName}/`,
[
{
name: 'GITHUB_SHA',
value: process.env.GITHUB_SHA,
},
],
[
{
ParameterKey: 'GithubToken',
ParameterValue: buildParameters.githubToken,
},
],
);
core.info('Starting part 2/4 (build unity project)');
await this.run(
buildUid,
buildParameters.awsStackName,
baseImage.toString(),
['/bin/sh'],
[
'-c',
`
cp -r /${efsDirectoryName}/${buildUid}/builder/dist/default-build-script/ /UnityBuilderAction;
cp -r /${efsDirectoryName}/${buildUid}/builder/dist/entrypoint.sh /entrypoint.sh;
cp -r /${efsDirectoryName}/${buildUid}/builder/dist/steps/ /steps;
chmod -R +x /entrypoint.sh;
chmod -R +x /steps;
/entrypoint.sh;
`,
],
`/${efsDirectoryName}`,
`/${efsDirectoryName}/${buildUid}/${repositoryDirectoryName}/`,
[
{
name: 'ContainerMemory',
value: buildParameters.remoteBuildMemory,
},
{
name: 'ContainerCpu',
value: buildParameters.remoteBuildCpu,
},
{
name: 'GITHUB_WORKSPACE',
value: `/${efsDirectoryName}/${buildUid}/${repositoryDirectoryName}/`,
},
{
name: 'PROJECT_PATH',
value: buildParameters.projectPath,
},
{
name: 'BUILD_PATH',
value: buildParameters.buildPath,
},
{
name: 'BUILD_FILE',
value: buildParameters.buildFile,
},
{
name: 'BUILD_NAME',
value: buildParameters.buildName,
},
{
name: 'BUILD_METHOD',
value: buildParameters.buildMethod,
},
{
name: 'CUSTOM_PARAMETERS',
value: buildParameters.customParameters,
},
{
name: 'CHOWN_FILES_TO',
value: buildParameters.chownFilesTo,
},
{
name: 'BUILD_TARGET',
value: buildParameters.platform,
},
{
name: 'ANDROID_VERSION_CODE',
value: buildParameters.androidVersionCode.toString(),
},
{
name: 'ANDROID_KEYSTORE_NAME',
value: buildParameters.androidKeystoreName,
},
{
name: 'ANDROID_KEYALIAS_NAME',
value: buildParameters.androidKeyaliasName,
},
],
[
{
ParameterKey: 'GithubToken',
ParameterValue: buildParameters.githubToken,
},
{
ParameterKey: 'UnityLicense',
ParameterValue: process.env.UNITY_LICENSE ? process.env.UNITY_LICENSE : '0',
},
{
ParameterKey: 'UnityEmail',
ParameterValue: process.env.UNITY_EMAIL ? process.env.UNITY_EMAIL : '0',
},
{
ParameterKey: 'UnityPassword',
ParameterValue: process.env.UNITY_PASSWORD ? process.env.UNITY_PASSWORD : '0',
},
{
ParameterKey: 'UnitySerial',
ParameterValue: process.env.UNITY_SERIAL ? process.env.UNITY_SERIAL : '0',
},
{
ParameterKey: 'AndroidKeystoreBase64',
ParameterValue: buildParameters.androidKeystoreBase64 ? buildParameters.androidKeystoreBase64 : '0',
},
{
ParameterKey: 'AndroidKeystorePass',
ParameterValue: buildParameters.androidKeystorePass ? buildParameters.androidKeystorePass : '0',
},
{
ParameterKey: 'AndroidKeyAliasPass',
ParameterValue: buildParameters.androidKeyaliasPass ? buildParameters.androidKeyaliasPass : '0',
},
],
);
core.info('Starting part 3/4 (zip unity build and Library for caching)');
// Cleanup
await this.run(
buildUid,
buildParameters.awsStackName,
'alpine',
['/bin/sh'],
[
'-c',
`
apk update
apk add zip
cd Library
zip -q -r lib-${buildUid}.zip .*
mv lib-${buildUid}.zip /${efsDirectoryName}/${cacheDirectoryName}/${branchName}/lib-${buildUid}.zip
cd ../../
zip -q -r build-${buildUid}.zip ${buildParameters.buildPath}/*
mv build-${buildUid}.zip /${efsDirectoryName}/${buildUid}/build-${buildUid}.zip
`,
],
`/${efsDirectoryName}`,
`/${efsDirectoryName}/${buildUid}/${repositoryDirectoryName}/${buildParameters.projectPath}`,
[
{
name: 'GITHUB_SHA',
value: process.env.GITHUB_SHA,
},
],
[
{
ParameterKey: 'GithubToken',
ParameterValue: buildParameters.githubToken,
},
],
);
core.info('Starting part 4/4 (upload build to s3)');
await this.run(
buildUid,
buildParameters.awsStackName,
'amazon/aws-cli',
['/bin/sh'],
[
'-c',
`
aws s3 cp ${buildUid}/build-${buildUid}.zip s3://game-ci-storage/
# no need to upload Library cache for now
# aws s3 cp /${efsDirectoryName}/${cacheDirectoryName}/${branchName}/lib-${buildUid}.zip s3://game-ci-storage/
rm -r ${buildUid}
`,
],
`/${efsDirectoryName}`,
`/${efsDirectoryName}/`,
[
{
name: 'GITHUB_SHA',
value: process.env.GITHUB_SHA,
},
{
name: 'AWS_DEFAULT_REGION',
value: process.env.AWS_DEFAULT_REGION,
},
],
[
{
ParameterKey: 'GithubToken',
ParameterValue: buildParameters.githubToken,
},
{
ParameterKey: 'AWSAccessKeyID',
ParameterValue: process.env.AWS_ACCESS_KEY_ID,
},
{
ParameterKey: 'AWSSecretAccessKey',
ParameterValue: process.env.AWS_SECRET_ACCESS_KEY,
},
],
);
} catch (error) {
core.setFailed(error);
core.error(error);
}
}
static async run(
buildUid: string,
stackName: string,
image: string,
entrypoint: string[],
commands,
mountdir,
workingdir,
environment,
secrets,
) {
const ECS = new SDK.ECS();
const CF = new SDK.CloudFormation();
const taskDef = await this.setupCloudFormations(
CF,
buildUid,
stackName,
image,
entrypoint,
commands,
mountdir,
workingdir,
secrets,
);
await this.runTask(taskDef, ECS, CF, environment, buildUid);
await this.cleanupResources(CF, taskDef);
}
static async setupCloudFormations(
CF,
buildUid: string,
stackName: string,
image: string,
entrypoint: string[],
commands,
mountdir,
workingdir,
secrets,
) {
const logid = customAlphabet(alphabet, 9)();
commands[1] += `
echo "${logid}"
`;
const taskDefStackName = `${stackName}-${buildUid}`;
const taskDefCloudFormation = fs.readFileSync(`${__dirname}/cloud-formations/task-def-formation.yml`, 'utf8');
await CF.createStack({
StackName: taskDefStackName,
TemplateBody: taskDefCloudFormation,
Parameters: [
{
ParameterKey: 'ImageUrl',
ParameterValue: image,
},
{
ParameterKey: 'ServiceName',
ParameterValue: taskDefStackName,
},
{
ParameterKey: 'Command',
ParameterValue: commands.join(','),
},
{
ParameterKey: 'EntryPoint',
ParameterValue: entrypoint.join(','),
},
{
ParameterKey: 'WorkingDirectory',
ParameterValue: workingdir,
},
{
ParameterKey: 'EFSMountDirectory',
ParameterValue: mountdir,
},
{
ParameterKey: 'BUILDID',
ParameterValue: buildUid,
},
...secrets,
],
}).promise();
core.info('Creating worker cluster...');
const cleanupTaskDefStackName = `${taskDefStackName}-cleanup`;
const cleanupCloudFormation = fs.readFileSync(`${__dirname}/cloud-formations/cloudformation-stack-ttl.yml`, 'utf8');
await CF.createStack({
StackName: cleanupTaskDefStackName,
TemplateBody: cleanupCloudFormation,
Capabilities: ['CAPABILITY_IAM'],
Parameters: [
{
ParameterKey: 'StackName',
ParameterValue: taskDefStackName,
},
{
ParameterKey: 'DeleteStackName',
ParameterValue: cleanupTaskDefStackName,
},
{
ParameterKey: 'TTL',
ParameterValue: '100',
},
{
ParameterKey: 'BUILDID',
ParameterValue: buildUid,
},
],
}).promise();
core.info('Creating cleanup cluster...');
try {
await CF.waitFor('stackCreateComplete', { StackName: taskDefStackName }).promise();
} catch (error) {
core.error(error);
}
const taskDefResources = await CF.describeStackResources({
StackName: taskDefStackName,
}).promise();
const baseResources = await CF.describeStackResources({ StackName: stackName }).promise();
// in the future we should offer a parameter to choose if you want the guarnteed shutdown.
core.info('Worker cluster created successfully (skipping wait for cleanup cluster to be ready)');
return {
taskDefStackName,
taskDefCloudFormation,
taskDefStackNameTTL: cleanupTaskDefStackName,
ttlCloudFormation: cleanupCloudFormation,
taskDefResources,
baseResources,
logid,
};
}
static async runTask(taskDef, ECS, CF, environment, buildUid) {
const cluster =
taskDef.baseResources.StackResources?.find((x) => x.LogicalResourceId === 'ECSCluster')?.PhysicalResourceId || '';
const taskDefinition =
taskDef.taskDefResources.StackResources?.find((x) => x.LogicalResourceId === 'TaskDefinition')
?.PhysicalResourceId || '';
const SubnetOne =
taskDef.baseResources.StackResources?.find((x) => x.LogicalResourceId === 'PublicSubnetOne')
?.PhysicalResourceId || '';
const SubnetTwo =
taskDef.baseResources.StackResources?.find((x) => x.LogicalResourceId === 'PublicSubnetTwo')
?.PhysicalResourceId || '';
const ContainerSecurityGroup =
taskDef.baseResources.StackResources?.find((x) => x.LogicalResourceId === 'ContainerSecurityGroup')
?.PhysicalResourceId || '';
const streamName =
taskDef.taskDefResources.StackResources?.find((x) => x.LogicalResourceId === 'KinesisStream')
?.PhysicalResourceId || '';
const task = await ECS.runTask({
cluster,
taskDefinition,
platformVersion: '1.4.0',
overrides: {
containerOverrides: [
{
name: taskDef.taskDefStackName,
environment: [...environment, { name: 'BUILDID', value: buildUid }],
},
],
},
launchType: 'FARGATE',
networkConfiguration: {
awsvpcConfiguration: {
subnets: [SubnetOne, SubnetTwo],
assignPublicIp: 'ENABLED',
securityGroups: [ContainerSecurityGroup],
},
},
}).promise();
core.info('Task is starting on worker cluster');
const taskArn = task.tasks?.[0].taskArn || '';
try {
await ECS.waitFor('tasksRunning', { tasks: [taskArn], cluster }).promise();
} catch (error) {
await new Promise((resolve) => setTimeout(resolve, 3000));
const describeTasks = await ECS.describeTasks({
tasks: [taskArn],
cluster,
}).promise();
core.info(`Task has ended ${describeTasks.tasks?.[0].containers?.[0].lastStatus}`);
core.setFailed(error);
core.error(error);
}
core.info(`Task is running on worker cluster`);
await this.streamLogsUntilTaskStops(ECS, CF, taskDef, cluster, taskArn, streamName);
await ECS.waitFor('tasksStopped', { cluster, tasks: [taskArn] }).promise();
const exitCode = (
await ECS.describeTasks({
tasks: [taskArn],
cluster,
}).promise()
).tasks?.[0].containers?.[0].exitCode;
if (exitCode !== 0) {
try {
await this.cleanupResources(CF, taskDef);
} catch (error) {
core.warning(`failed to cleanup ${error}`);
}
core.error(`job failed with exit code ${exitCode}`);
throw new Error(`job failed with exit code ${exitCode}`);
} else {
core.info(`Task has finished successfully`);
}
}
static async streamLogsUntilTaskStops(ECS: AWS.ECS, CF, taskDef, clusterName, taskArn, kinesisStreamName) {
// watching logs
const kinesis = new SDK.Kinesis();
const getTaskData = async () => {
const tasks = await ECS.describeTasks({
cluster: clusterName,
tasks: [taskArn],
}).promise();
return tasks.tasks?.[0];
};
const stream = await kinesis
.describeStream({
StreamName: kinesisStreamName,
})
.promise();
let iterator =
(
await kinesis
.getShardIterator({
ShardIteratorType: 'TRIM_HORIZON',
StreamName: stream.StreamDescription.StreamName,
ShardId: stream.StreamDescription.Shards[0].ShardId,
})
.promise()
).ShardIterator || '';
await CF.waitFor('stackCreateComplete', { StackName: taskDef.taskDefStackNameTTL }).promise();
core.info(`Task status is ${(await getTaskData())?.lastStatus}`);
const logBaseUrl = `https://${SDK.config.region}.console.aws.amazon.com/cloudwatch/home?region=${SDK.config.region}#logsV2:log-groups/log-group/${taskDef.taskDefStackName}`;
core.info(`You can also see the logs at AWS Cloud Watch: ${logBaseUrl}`);
let readingLogs = true;
let timestamp: number = 0;
while (readingLogs) {
await new Promise((resolve) => setTimeout(resolve, 1500));
const taskData = await getTaskData();
if (taskData?.lastStatus !== 'RUNNING') {
if (timestamp === 0) {
core.info('Task stopped, streaming end of logs');
timestamp = Date.now();
}
if (timestamp !== 0 && Date.now() - timestamp < 30000) {
core.info('Task status is not RUNNING for 30 seconds, last query for logs');
readingLogs = false;
}
}
const records = await kinesis
.getRecords({
ShardIterator: iterator,
})
.promise();
iterator = records.NextShardIterator || '';
if (records.Records.length > 0 && iterator) {
for (let index = 0; index < records.Records.length; index++) {
const json = JSON.parse(
zlib.gunzipSync(Buffer.from(records.Records[index].Data as string, 'base64')).toString('utf8'),
);
if (json.messageType === 'DATA_MESSAGE') {
for (let logEventsIndex = 0; logEventsIndex < json.logEvents.length; logEventsIndex++) {
if (json.logEvents[logEventsIndex].message.includes(taskDef.logid)) {
core.info('End of task logs');
readingLogs = false;
} else {
core.info(json.logEvents[logEventsIndex].message);
}
}
}
}
}
}
}
static async cleanupResources(CF, taskDef) {
await CF.deleteStack({
StackName: taskDef.taskDefStackName,
}).promise();
await CF.deleteStack({
StackName: taskDef.taskDefStackNameTTL,
}).promise();
await CF.waitFor('stackDeleteComplete', {
StackName: taskDef.taskDefStackName,
}).promise();
// Currently too slow and causes too much waiting
await CF.waitFor('stackDeleteComplete', {
StackName: taskDef.taskDefStackNameTTL,
}).promise();
core.info('Cleanup complete');
}
static onlog(batch) {
for (const log of batch) {
core.info(`log: ${log}`);
}
}
}
export default AWS;

View File

@ -5,7 +5,34 @@ import UnityVersioning from './unity-versioning';
import Versioning from './versioning';
class BuildParameters {
static async create() {
public version!: string;
public customImage!: string;
public runnerTempPath: string | undefined;
public platform!: string;
public projectPath!: string;
public buildName!: string;
public buildPath!: string;
public buildFile!: string;
public buildMethod!: string;
public buildVersion!: string;
public androidVersionCode!: string;
public androidKeystoreName!: string;
public androidKeystoreBase64!: string;
public androidKeystorePass!: string;
public androidKeyaliasName!: string;
public androidKeyaliasPass!: string;
public customParameters!: string;
public remoteBuildCluster!: string;
public awsStackName!: string;
public kubeConfig!: string;
public githubToken!: string;
public remoteBuildMemory!: string;
public remoteBuildCpu!: string;
public kubeVolumeSize!: string;
public kubeVolume!: string;
public chownFilesTo!: string;
static async create(): Promise<BuildParameters> {
const buildFile = this.parseBuildFile(Input.buildName, Input.targetPlatform, Input.androidAppBundle);
const unityVersion = UnityVersioning.determineUnityVersion(Input.projectPath, Input.unityVersion);

View File

@ -10,7 +10,7 @@ import Project from './project';
import Unity from './unity';
import Versioning from './versioning';
import Kubernetes from './kubernetes';
import AWS from './aws';
import RemoteBuilder from './remote-builder/remote-builder';
export {
Action,
@ -25,5 +25,5 @@ export {
Unity,
Versioning,
Kubernetes,
AWS,
RemoteBuilder,
};

View File

@ -0,0 +1,253 @@
import * as SDK from 'aws-sdk';
import { customAlphabet } from 'nanoid';
import RemoteBuilderSecret from './remote-builder-secret';
import RemoteBuilderEnvironmentVariable from './remote-builder-environment-variable';
import * as fs from 'fs';
import * as core from '@actions/core';
import RemoteBuilderTaskDef from './remote-builder-task-def';
import RemoteBuilderConstants from './remote-builder-constants';
import AWSBuildRunner from './aws-build-runner';
class AWSBuildEnvironment {
static async runBuild(
buildId: string,
stackName: string,
image: string,
commands: string[],
mountdir: string,
workingdir: string,
environment: RemoteBuilderEnvironmentVariable[],
secrets: RemoteBuilderSecret[],
) {
const ECS = new SDK.ECS();
const CF = new SDK.CloudFormation();
const entrypoint = ['/bin/sh'];
const taskDef = await this.setupCloudFormations(
CF,
buildId,
stackName,
image,
entrypoint,
commands,
mountdir,
workingdir,
secrets,
);
try {
await AWSBuildRunner.runTask(taskDef, ECS, CF, environment, buildId);
} finally {
await this.cleanupResources(CF, taskDef);
}
}
// static async setupPlatformResources() {
// throw new Error('Method not implemented.');
// }
static getParameterTemplate(p1) {
return `
${p1}:
Type: String
Default: ''
`;
}
static getSecretTemplate(p1) {
return `
${p1}Secret:
Type: AWS::SecretsManager::Secret
Properties:
Name: !Join [ "", [ '${p1}', !Ref BUILDID ] ]
SecretString: !Ref ${p1}
`;
}
static getSecretDefinitionTemplate(p1, p2) {
return `
- Name: '${p1}'
ValueFrom: !Ref ${p2}Secret
`;
}
static insertAtTemplate(template, insertionKey, insertion) {
const index = template.search(insertionKey) + insertionKey.length + '\n'.length;
template = [template.slice(0, index), insertion, template.slice(index)].join('');
return template;
}
static async setupCloudFormations(
CF: SDK.CloudFormation,
buildUid: string,
stackName: string,
image: string,
entrypoint: string[],
commands: string[],
mountdir: string,
workingdir: string,
secrets: RemoteBuilderSecret[],
): Promise<RemoteBuilderTaskDef> {
const logid = customAlphabet(RemoteBuilderConstants.alphabet, 9)();
commands[1] += `
echo "${logid}"
`;
const taskDefStackName = `${stackName}-${buildUid}`;
let taskDefCloudFormation = this.readTaskCloudFormationTemplate();
const cleanupTaskDefStackName = `${taskDefStackName}-cleanup`;
const cleanupCloudFormation = fs.readFileSync(`${__dirname}/cloud-formations/cloudformation-stack-ttl.yml`, 'utf8');
try {
for (const secret of secrets) {
taskDefCloudFormation = this.insertAtTemplate(
taskDefCloudFormation,
'p1 - input',
this.getParameterTemplate(secret.ParameterKey.replace(/[^\dA-Za-z]/g, '')),
);
taskDefCloudFormation = this.insertAtTemplate(
taskDefCloudFormation,
'p2 - secret',
this.getSecretTemplate(secret.ParameterKey.replace(/[^\dA-Za-z]/g, '')),
);
taskDefCloudFormation = this.insertAtTemplate(
taskDefCloudFormation,
'p3 - container def',
this.getSecretDefinitionTemplate(secret.EnvironmentVariable, secret.ParameterKey.replace(/[^\dA-Za-z]/g, '')),
);
}
const mappedSecrets = secrets.map((x) => {
return { ParameterKey: x.ParameterKey.replace(/[^\dA-Za-z]/g, ''), ParameterValue: x.ParameterValue };
});
await CF.createStack({
StackName: taskDefStackName,
TemplateBody: taskDefCloudFormation,
Parameters: [
{
ParameterKey: 'ImageUrl',
ParameterValue: image,
},
{
ParameterKey: 'ServiceName',
ParameterValue: taskDefStackName,
},
{
ParameterKey: 'Command',
ParameterValue: commands.join(','),
},
{
ParameterKey: 'EntryPoint',
ParameterValue: entrypoint.join(','),
},
{
ParameterKey: 'WorkingDirectory',
ParameterValue: workingdir,
},
{
ParameterKey: 'EFSMountDirectory',
ParameterValue: mountdir,
},
{
ParameterKey: 'BUILDID',
ParameterValue: buildUid,
},
...mappedSecrets,
],
}).promise();
core.info('Creating worker cluster...');
await CF.createStack({
StackName: cleanupTaskDefStackName,
TemplateBody: cleanupCloudFormation,
Capabilities: ['CAPABILITY_IAM'],
Parameters: [
{
ParameterKey: 'StackName',
ParameterValue: taskDefStackName,
},
{
ParameterKey: 'DeleteStackName',
ParameterValue: cleanupTaskDefStackName,
},
{
ParameterKey: 'TTL',
ParameterValue: '100',
},
{
ParameterKey: 'BUILDID',
ParameterValue: buildUid,
},
],
}).promise();
core.info('Creating cleanup cluster...');
await CF.waitFor('stackCreateComplete', { StackName: taskDefStackName }).promise();
} catch (error) {
await AWSBuildEnvironment.handleStackCreationFailure(error, CF, taskDefStackName, taskDefCloudFormation, secrets);
throw error;
}
const taskDefResources = (
await CF.describeStackResources({
StackName: taskDefStackName,
}).promise()
).StackResources;
const baseResources = (await CF.describeStackResources({ StackName: stackName }).promise()).StackResources;
// in the future we should offer a parameter to choose if you want the guarnteed shutdown.
core.info('Worker cluster created successfully (skipping wait for cleanup cluster to be ready)');
return {
taskDefStackName,
taskDefCloudFormation,
taskDefStackNameTTL: cleanupTaskDefStackName,
ttlCloudFormation: cleanupCloudFormation,
taskDefResources,
baseResources,
logid,
};
}
private static async handleStackCreationFailure(
error: any,
CF: SDK.CloudFormation,
taskDefStackName: string,
taskDefCloudFormation: string,
secrets: RemoteBuilderSecret[],
) {
core.info(JSON.stringify(secrets, undefined, 4));
core.info(taskDefCloudFormation);
const events = (await CF.describeStackEvents({ StackName: taskDefStackName }).promise()).StackEvents;
const resources = (await CF.describeStackResources({ StackName: taskDefStackName }).promise()).StackResources;
core.info(JSON.stringify(events, undefined, 4));
core.info(JSON.stringify(resources, undefined, 4));
core.error(error);
}
static readTaskCloudFormationTemplate(): string {
return fs.readFileSync(`${__dirname}/cloud-formations/task-def-formation.yml`, 'utf8');
}
static async cleanupResources(CF: SDK.CloudFormation, taskDef: RemoteBuilderTaskDef) {
core.info('Cleanup starting');
await CF.deleteStack({
StackName: taskDef.taskDefStackName,
}).promise();
await CF.deleteStack({
StackName: taskDef.taskDefStackNameTTL,
}).promise();
await CF.waitFor('stackDeleteComplete', {
StackName: taskDef.taskDefStackName,
}).promise();
// Currently too slow and causes too much waiting
await CF.waitFor('stackDeleteComplete', {
StackName: taskDef.taskDefStackNameTTL,
}).promise();
core.info('Cleanup complete');
}
}
export default AWSBuildEnvironment;

View File

@ -0,0 +1,165 @@
import * as AWS from 'aws-sdk';
import RemoteBuilderEnvironmentVariable from './remote-builder-environment-variable';
import * as core from '@actions/core';
import RemoteBuilderTaskDef from './remote-builder-task-def';
import * as zlib from 'zlib';
class AWSBuildRunner {
static async runTask(
taskDef: RemoteBuilderTaskDef,
ECS: AWS.ECS,
CF: AWS.CloudFormation,
environment: RemoteBuilderEnvironmentVariable[],
buildUid: string,
) {
const cluster = taskDef.baseResources?.find((x) => x.LogicalResourceId === 'ECSCluster')?.PhysicalResourceId || '';
const taskDefinition =
taskDef.taskDefResources?.find((x) => x.LogicalResourceId === 'TaskDefinition')?.PhysicalResourceId || '';
const SubnetOne =
taskDef.baseResources?.find((x) => x.LogicalResourceId === 'PublicSubnetOne')?.PhysicalResourceId || '';
const SubnetTwo =
taskDef.baseResources?.find((x) => x.LogicalResourceId === 'PublicSubnetTwo')?.PhysicalResourceId || '';
const ContainerSecurityGroup =
taskDef.baseResources?.find((x) => x.LogicalResourceId === 'ContainerSecurityGroup')?.PhysicalResourceId || '';
const streamName =
taskDef.taskDefResources?.find((x) => x.LogicalResourceId === 'KinesisStream')?.PhysicalResourceId || '';
const task = await ECS.runTask({
cluster,
taskDefinition,
platformVersion: '1.4.0',
overrides: {
containerOverrides: [
{
name: taskDef.taskDefStackName,
environment: [...environment, { name: 'BUILDID', value: buildUid }],
},
],
},
launchType: 'FARGATE',
networkConfiguration: {
awsvpcConfiguration: {
subnets: [SubnetOne, SubnetTwo],
assignPublicIp: 'ENABLED',
securityGroups: [ContainerSecurityGroup],
},
},
}).promise();
core.info('Task is starting on worker cluster');
const taskArn = task.tasks?.[0].taskArn || '';
try {
await ECS.waitFor('tasksRunning', { tasks: [taskArn], cluster }).promise();
} catch (error) {
await new Promise((resolve) => setTimeout(resolve, 3000));
const describeTasks = await ECS.describeTasks({
tasks: [taskArn],
cluster,
}).promise();
core.info(`Task has ended ${describeTasks.tasks?.[0].containers?.[0].lastStatus}`);
core.setFailed(error);
core.error(error);
}
core.info(`Task is running on worker cluster`);
await this.streamLogsUntilTaskStops(ECS, CF, taskDef, cluster, taskArn, streamName);
await ECS.waitFor('tasksStopped', { cluster, tasks: [taskArn] }).promise();
const exitCode = (
await ECS.describeTasks({
tasks: [taskArn],
cluster,
}).promise()
).tasks?.[0].containers?.[0].exitCode;
if (exitCode !== 0) {
core.error(`job failed with exit code ${exitCode}`);
throw new Error(`job failed with exit code ${exitCode}`);
} else {
core.info(`Task has finished successfully`);
}
}
static async streamLogsUntilTaskStops(
ECS: AWS.ECS,
CF: AWS.CloudFormation,
taskDef: RemoteBuilderTaskDef,
clusterName: string,
taskArn: string,
kinesisStreamName: string,
) {
// watching logs
const kinesis = new AWS.Kinesis();
const getTaskData = async () => {
const tasks = await ECS.describeTasks({
cluster: clusterName,
tasks: [taskArn],
}).promise();
return tasks.tasks?.[0];
};
const stream = await kinesis
.describeStream({
StreamName: kinesisStreamName,
})
.promise();
let iterator =
(
await kinesis
.getShardIterator({
ShardIteratorType: 'TRIM_HORIZON',
StreamName: stream.StreamDescription.StreamName,
ShardId: stream.StreamDescription.Shards[0].ShardId,
})
.promise()
).ShardIterator || '';
await CF.waitFor('stackCreateComplete', { StackName: taskDef.taskDefStackNameTTL }).promise();
core.info(`Task status is ${(await getTaskData())?.lastStatus}`);
const logBaseUrl = `https://${AWS.config.region}.console.aws.amazon.com/cloudwatch/home?region=${AWS.config.region}#logsV2:log-groups/log-group/${taskDef.taskDefStackName}`;
core.info(`You can also see the logs at AWS Cloud Watch: ${logBaseUrl}`);
let readingLogs = true;
let timestamp: number = 0;
while (readingLogs) {
await new Promise((resolve) => setTimeout(resolve, 1500));
const taskData = await getTaskData();
if (taskData?.lastStatus !== 'RUNNING') {
if (timestamp === 0) {
core.info('Task stopped, streaming end of logs');
timestamp = Date.now();
}
if (timestamp !== 0 && Date.now() - timestamp < 30000) {
core.info('Task status is not RUNNING for 30 seconds, last query for logs');
readingLogs = false;
}
}
const records = await kinesis
.getRecords({
ShardIterator: iterator,
})
.promise();
iterator = records.NextShardIterator || '';
if (records.Records.length > 0 && iterator) {
for (let index = 0; index < records.Records.length; index++) {
const json = JSON.parse(
zlib.gunzipSync(Buffer.from(records.Records[index].Data as string, 'base64')).toString('utf8'),
);
if (json.messageType === 'DATA_MESSAGE') {
for (let logEventsIndex = 0; logEventsIndex < json.logEvents.length; logEventsIndex++) {
if (json.logEvents[logEventsIndex].message.includes(taskDef.logid)) {
core.info('End of task logs');
readingLogs = false;
} else {
core.info(json.logEvents[logEventsIndex].message);
}
}
}
}
}
}
}
}
export default AWSBuildRunner;

View File

@ -0,0 +1,4 @@
class RemoteBuilderConstants {
static alphabet = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz';
}
export default RemoteBuilderConstants;

View File

@ -0,0 +1,5 @@
class RemoteBuilderEnvironmentVariable {
public name!: string;
public value!: string;
}
export default RemoteBuilderEnvironmentVariable;

View File

@ -0,0 +1,6 @@
class RemoteBuilderSecret {
public ParameterKey!: string;
public EnvironmentVariable!: string;
public ParameterValue!: string;
}
export default RemoteBuilderSecret;

View File

@ -0,0 +1,12 @@
import * as AWS from 'aws-sdk';
class RemoteBuilderTaskDef {
public taskDefStackName!: string;
public taskDefCloudFormation!: string;
public taskDefStackNameTTL!: string;
public ttlCloudFormation!: string;
public taskDefResources: AWS.CloudFormation.StackResources | undefined;
public baseResources: AWS.CloudFormation.StackResources | undefined;
public logid!: string;
}
export default RemoteBuilderTaskDef;

View File

@ -0,0 +1,419 @@
import { customAlphabet } from 'nanoid';
import AWSBuildPlatform from './aws-build-platform';
import * as core from '@actions/core';
import RemoteBuilderConstants from './remote-builder-constants';
import { BuildParameters } from '..';
const repositoryDirectoryName = 'repo';
const efsDirectoryName = 'data';
const cacheDirectoryName = 'cache';
class RemoteBuilder {
static SteamDeploy: boolean = false;
static async build(buildParameters: BuildParameters, baseImage) {
try {
this.SteamDeploy = process.env.STEAM_DEPLOY !== undefined || false;
const nanoid = customAlphabet(RemoteBuilderConstants.alphabet, 4);
const buildUid = `${process.env.GITHUB_RUN_NUMBER}-${buildParameters.platform
.replace('Standalone', '')
.replace('standalone', '')}-${nanoid()}`;
const defaultBranchName =
process.env.GITHUB_REF?.split('/')
.filter((x) => {
x = x[0].toUpperCase() + x.slice(1);
return x;
})
.join('') || '';
const branchName =
process.env.REMOTE_BUILDER_CACHE !== undefined ? process.env.REMOTE_BUILDER_CACHE : defaultBranchName;
const token: string = buildParameters.githubToken;
const defaultSecretsArray = [
{
ParameterKey: 'GithubToken',
EnvironmentVariable: 'GITHUB_TOKEN',
ParameterValue: token,
},
];
await RemoteBuilder.SetupStep(buildUid, buildParameters, branchName, defaultSecretsArray);
await RemoteBuilder.BuildStep(buildUid, buildParameters, baseImage, defaultSecretsArray);
await RemoteBuilder.CompressionStep(buildUid, buildParameters, branchName, defaultSecretsArray);
await RemoteBuilder.UploadArtifacts(buildUid, buildParameters, branchName, defaultSecretsArray);
if (this.SteamDeploy) {
await RemoteBuilder.DeployToSteam(buildUid, buildParameters, defaultSecretsArray);
}
} catch (error) {
core.setFailed(error);
core.error(error);
}
}
private static async SetupStep(
buildUid: string,
buildParameters: BuildParameters,
branchName: string | undefined,
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
) {
core.info('Starting step 1/4 clone and restore cache)');
await AWSBuildPlatform.runBuild(
buildUid,
buildParameters.awsStackName,
'alpine/git',
[
'-c',
`apk update;
apk add unzip;
apk add git-lfs;
apk add jq;
# Get source repo for project to be built and game-ci repo for utilties
git clone https://${buildParameters.githubToken}@github.com/${
process.env.GITHUB_REPOSITORY
}.git ${buildUid}/${repositoryDirectoryName} -q
git clone https://${buildParameters.githubToken}@github.com/game-ci/unity-builder.git ${buildUid}/builder -q
git clone https://${buildParameters.githubToken}@github.com/game-ci/steam-deploy.git ${buildUid}/steam -q
cd /${efsDirectoryName}/${buildUid}/${repositoryDirectoryName}/
git checkout $GITHUB_SHA
cd /${efsDirectoryName}/
# Look for usable cache
if [ ! -d ${cacheDirectoryName} ]; then
mkdir ${cacheDirectoryName}
fi
cd ${cacheDirectoryName}
if [ ! -d "${branchName}" ]; then
mkdir "${branchName}"
fi
cd "${branchName}"
echo ''
echo "Cached Libraries for ${branchName} from previous builds:"
ls
echo ''
ls "/${efsDirectoryName}/${buildUid}/${repositoryDirectoryName}/${buildParameters.projectPath}"
libDir="/${efsDirectoryName}/${buildUid}/${repositoryDirectoryName}/${buildParameters.projectPath}/Library"
if [ -d "$libDir" ]; then
rm -r "$libDir"
echo "Setup .gitignore to ignore Library folder and remove it from builds"
fi
echo 'Checking cache'
# Restore cache
latest=$(ls -t | head -1)
if [ ! -z "$latest" ]; then
echo "Library cache exists from build $latest from ${branchName}"
echo 'Creating empty Library folder for cache'
mkdir $libDir
unzip -q $latest -d $libDir
# purge cache
${process.env.PURGE_REMOTE_BUILDER_CACHE === undefined ? '#' : ''} rm -r $libDir
else
echo 'Cache does not exist'
fi
# Print out important directories
echo ''
echo 'Repo:'
ls /${efsDirectoryName}/${buildUid}/${repositoryDirectoryName}/
echo ''
echo 'Project:'
ls /${efsDirectoryName}/${buildUid}/${repositoryDirectoryName}/${buildParameters.projectPath}
echo ''
echo 'Library:'
ls /${efsDirectoryName}/${buildUid}/${repositoryDirectoryName}/${buildParameters.projectPath}/Library/
echo ''
`,
],
`/${efsDirectoryName}`,
`/${efsDirectoryName}/`,
[
{
name: 'GITHUB_SHA',
value: process.env.GITHUB_SHA || '',
},
],
defaultSecretsArray,
);
}
private static async BuildStep(
buildUid: string,
buildParameters: BuildParameters,
baseImage: any,
defaultSecretsArray: any[],
) {
const buildSecrets = new Array();
buildSecrets.push(...defaultSecretsArray);
if (process.env.UNITY_LICENSE)
buildSecrets.push({
ParameterKey: 'UnityLicense',
EnvironmentVariable: 'UNITY_LICENSE',
ParameterValue: process.env.UNITY_LICENSE,
});
if (process.env.UNITY_EMAIL)
buildSecrets.push({
ParameterKey: 'UnityEmail',
EnvironmentVariable: 'UNITY_EMAIL',
ParameterValue: process.env.UNITY_EMAIL,
});
if (process.env.UNITY_PASSWORD)
buildSecrets.push({
ParameterKey: 'UnityPassword',
EnvironmentVariable: 'UNITY_PASSWORD',
ParameterValue: process.env.UNITY_PASSWORD,
});
if (process.env.UNITY_SERIAL)
buildSecrets.push({
ParameterKey: 'UnitySerial',
EnvironmentVariable: 'UNITY_SERIAL',
ParameterValue: process.env.UNITY_SERIAL,
});
if (buildParameters.androidKeystoreBase64)
buildSecrets.push({
ParameterKey: 'AndroidKeystoreBase64',
EnvironmentVariable: 'ANDROID_KEYSTORE_BASE64',
ParameterValue: buildParameters.androidKeystoreBase64,
});
if (buildParameters.androidKeystorePass)
buildSecrets.push({
ParameterKey: 'AndroidKeystorePass',
EnvironmentVariable: 'ANDROID_KEYSTORE_PASS',
ParameterValue: buildParameters.androidKeystorePass,
});
if (buildParameters.androidKeyaliasPass)
buildSecrets.push({
ParameterKey: 'AndroidKeyAliasPass',
EnvironmentVariable: 'AWS_ACCESS_KEY_ALIAS_PASS',
ParameterValue: buildParameters.androidKeyaliasPass,
});
core.info('Starting part 2/4 (build unity project)');
await AWSBuildPlatform.runBuild(
buildUid,
buildParameters.awsStackName,
baseImage.toString(),
[
'-c',
`
cp -r /${efsDirectoryName}/${buildUid}/builder/dist/default-build-script/ /UnityBuilderAction;
cp -r /${efsDirectoryName}/${buildUid}/builder/dist/entrypoint.sh /entrypoint.sh;
cp -r /${efsDirectoryName}/${buildUid}/builder/dist/steps/ /steps;
chmod -R +x /entrypoint.sh;
chmod -R +x /steps;
/entrypoint.sh;
`,
],
`/${efsDirectoryName}`,
`/${efsDirectoryName}/${buildUid}/${repositoryDirectoryName}/`,
[
{
name: 'ContainerMemory',
value: buildParameters.remoteBuildMemory,
},
{
name: 'ContainerCpu',
value: buildParameters.remoteBuildCpu,
},
{
name: 'GITHUB_WORKSPACE',
value: `/${efsDirectoryName}/${buildUid}/${repositoryDirectoryName}/`,
},
{
name: 'PROJECT_PATH',
value: buildParameters.projectPath,
},
{
name: 'BUILD_PATH',
value: buildParameters.buildPath,
},
{
name: 'BUILD_FILE',
value: buildParameters.buildFile,
},
{
name: 'BUILD_NAME',
value: buildParameters.buildName,
},
{
name: 'BUILD_METHOD',
value: buildParameters.buildMethod,
},
{
name: 'CUSTOM_PARAMETERS',
value: buildParameters.customParameters,
},
{
name: 'BUILD_TARGET',
value: buildParameters.platform,
},
{
name: 'ANDROID_VERSION_CODE',
value: buildParameters.androidVersionCode.toString(),
},
{
name: 'ANDROID_KEYSTORE_NAME',
value: buildParameters.androidKeystoreName,
},
{
name: 'ANDROID_KEYALIAS_NAME',
value: buildParameters.androidKeyaliasName,
},
],
buildSecrets,
);
}
private static async CompressionStep(
buildUid: string,
buildParameters: BuildParameters,
branchName: string | undefined,
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
) {
core.info('Starting step 3/4 build compression');
// Cleanup
await AWSBuildPlatform.runBuild(
buildUid,
buildParameters.awsStackName,
'alpine',
[
'-c',
`
apk update
apk add zip
cd Library
zip -r lib-${buildUid}.zip .*
mv lib-${buildUid}.zip /${efsDirectoryName}/${cacheDirectoryName}/${branchName}/lib-${buildUid}.zip
cd ../../
zip -r build-${buildUid}.zip ${buildParameters.buildPath}/*
mv build-${buildUid}.zip /${efsDirectoryName}/${buildUid}/build-${buildUid}.zip
`,
],
`/${efsDirectoryName}`,
`/${efsDirectoryName}/${buildUid}/${repositoryDirectoryName}/${buildParameters.projectPath}`,
[
{
name: 'GITHUB_SHA',
value: process.env.GITHUB_SHA || '',
},
],
defaultSecretsArray,
);
core.info('compression step complete');
}
private static async UploadArtifacts(
buildUid: string,
buildParameters: BuildParameters,
branchName: string | undefined,
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
) {
core.info('Starting step 4/4 upload build to s3');
await AWSBuildPlatform.runBuild(
buildUid,
buildParameters.awsStackName,
'amazon/aws-cli',
[
'-c',
`
aws s3 cp ${buildUid}/build-${buildUid}.zip s3://game-ci-storage/
# no need to upload Library cache for now
# aws s3 cp /${efsDirectoryName}/${cacheDirectoryName}/${branchName}/lib-${buildUid}.zip s3://game-ci-storage/
${this.SteamDeploy ? '#' : ''} rm -r ${buildUid}
`,
],
`/${efsDirectoryName}`,
`/${efsDirectoryName}/`,
[
{
name: 'GITHUB_SHA',
value: process.env.GITHUB_SHA || '',
},
{
name: 'AWS_DEFAULT_REGION',
value: process.env.AWS_DEFAULT_REGION || '',
},
],
[
{
ParameterKey: 'AWSAccessKeyID',
EnvironmentVariable: 'AWS_ACCESS_KEY_ID',
ParameterValue: process.env.AWS_ACCESS_KEY_ID || '',
},
{
ParameterKey: 'AWSSecretAccessKey',
EnvironmentVariable: 'AWS_SECRET_ACCESS_KEY',
ParameterValue: process.env.AWS_SECRET_ACCESS_KEY || '',
},
...defaultSecretsArray,
],
);
}
private static async DeployToSteam(
buildUid: string,
buildParameters: BuildParameters,
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
) {
core.info('Starting steam deployment');
await AWSBuildPlatform.runBuild(
buildUid,
buildParameters.awsStackName,
'cm2network/steamcmd:root',
[
'-c',
`
ls
ls /
cp -r /${efsDirectoryName}/${buildUid}/steam/action/entrypoint.sh /entrypoint.sh;
cp -r /${efsDirectoryName}/${buildUid}/steam/action/steps/ /steps;
chmod -R +x /entrypoint.sh;
chmod -R +x /steps;
/entrypoint.sh;
rm -r /${efsDirectoryName}/${buildUid}
`,
],
`/${efsDirectoryName}`,
`/${efsDirectoryName}/${buildUid}/steam/action/`,
[
{
name: 'GITHUB_SHA',
value: process.env.GITHUB_SHA || '',
},
],
[
{
EnvironmentVariable: 'INPUT_APPID',
ParameterKey: 'appId',
ParameterValue: process.env.APP_ID || '',
},
{
EnvironmentVariable: 'INPUT_BUILDDESCRIPTION',
ParameterKey: 'buildDescription',
ParameterValue: process.env.BUILD_DESCRIPTION || '',
},
{
EnvironmentVariable: 'INPUT_ROOTPATH',
ParameterKey: 'rootPath',
ParameterValue: process.env.ROOT_PATH || '',
},
{
EnvironmentVariable: 'INPUT_RELEASEBRANCH',
ParameterKey: 'releaseBranch',
ParameterValue: process.env.RELEASE_BRANCH || '',
},
{
EnvironmentVariable: 'INPUT_LOCALCONTENTSERVER',
ParameterKey: 'localContentServer',
ParameterValue: process.env.LOCAL_CONTENT_SERVER || '',
},
{
EnvironmentVariable: 'INPUT_PREVIEWENABLED',
ParameterKey: 'previewEnabled',
ParameterValue: process.env.PREVIEW_ENABLED || '',
},
...defaultSecretsArray,
],
);
}
}
export default RemoteBuilder;