EthanPisani/unity-builder
1
0
Fork 0
mirror of https://github.com/game-ci/unity-builder.git synced 2025-07-04 12:25:19 -04:00
Code Issues Packages Projects Releases Wiki Activity

chore(aws): refactor aws runner to support SDK v3

Browse Source
This commit is contained in:
Matheus Costa 2025-04-02 21:04:15 -03:00
parent ac08240d19
commit c041900a08
No known key found for this signature in database
GPG Key ID: 3574E997C5D4563B
7 changed files with 147 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
dist/index.js.map generated vendored
View File

Binary file not shown.

56
src/model/cloud-runner/providers/aws/aws-base-stack.ts
View File

@ -1,6 +1,18 @@
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import * as core from '@actions/core';
import * as SDK from 'aws-sdk';
import {
CloudFormation,
CreateStackCommand,
CreateStackCommandInput,
DescribeStacksCommand,
DescribeStacksCommandInput,
ListStacksCommand,
Parameter,
UpdateStackCommand,
UpdateStackCommandInput,
waitUntilStackCreateComplete,
waitUntilStackUpdateComplete,
} from '@aws-sdk/client-cloudformation';
import { BaseStackFormation } from './cloud-formations/base-stack-formation';
import crypto from 'node:crypto';
@ -10,51 +22,49 @@ export class AWSBaseStack {
}
private baseStackName: string;
async setupBaseStack(CF: SDK.CloudFormation) {
async setupBaseStack(CF: CloudFormation) {
const baseStackName = this.baseStackName;
const baseStack = BaseStackFormation.formation;
// Cloud Formation Input
const describeStackInput: SDK.CloudFormation.DescribeStacksInput = {
const describeStackInput: DescribeStacksCommandInput = {
StackName: baseStackName,
};
const parametersWithoutHash: SDK.CloudFormation.Parameter[] = [
{ ParameterKey: 'EnvironmentName', ParameterValue: baseStackName },
];
const parametersWithoutHash: Parameter[] = [{ ParameterKey: 'EnvironmentName', ParameterValue: baseStackName }];
const parametersHash = crypto
.createHash('md5')
.update(baseStack + JSON.stringify(parametersWithoutHash))
.digest('hex');
const parameters: SDK.CloudFormation.Parameter[] = [
const parameters: Parameter[] = [
...parametersWithoutHash,
...[{ ParameterKey: 'Version', ParameterValue: parametersHash }],
];
const updateInput: SDK.CloudFormation.UpdateStackInput = {
const updateInput: UpdateStackCommandInput = {
StackName: baseStackName,
TemplateBody: baseStack,
Parameters: parameters,
Capabilities: ['CAPABILITY_IAM'],
};
const createStackInput: SDK.CloudFormation.CreateStackInput = {
const createStackInput: CreateStackCommandInput = {
StackName: baseStackName,
TemplateBody: baseStack,
Parameters: parameters,
Capabilities: ['CAPABILITY_IAM'],
};
const stacks = await CF.listStacks({
StackStatusFilter: ['UPDATE_COMPLETE', 'CREATE_COMPLETE', 'ROLLBACK_COMPLETE'],
}).promise();
const stacks = await CF.send(
new ListStacksCommand({ StackStatusFilter: ['UPDATE_COMPLETE', 'CREATE_COMPLETE', 'ROLLBACK_COMPLETE'] }),
);
const stackNames = stacks.StackSummaries?.map((x) => x.StackName) || [];
const stackExists: Boolean = stackNames.includes(baseStackName) || false;
const describeStack = async () => {
return await CF.describeStacks(describeStackInput).promise();
return await CF.send(new DescribeStacksCommand(describeStackInput));
};
try {
if (!stackExists) {
CloudRunnerLogger.log(`${baseStackName} stack does not exist (${JSON.stringify(stackNames)})`);
await CF.createStack(createStackInput).promise();
await CF.send(new CreateStackCommand(createStackInput));
CloudRunnerLogger.log(`created stack (version: ${parametersHash})`);
}
const CFState = await describeStack();
@ -65,7 +75,13 @@ export class AWSBaseStack {
const stackVersion = stack.Parameters?.find((x) => x.ParameterKey === 'Version')?.ParameterValue;
if (stack.StackStatus === 'CREATE_IN_PROGRESS') {
await CF.waitFor('stackCreateComplete', describeStackInput).promise();
await waitUntilStackCreateComplete(
{
client: CF,
maxWaitTime: 200,
},
describeStackInput,
);
}
if (stackExists) {
@ -73,7 +89,7 @@ export class AWSBaseStack {
if (parametersHash !== stackVersion) {
CloudRunnerLogger.log(`Attempting update of base stack`);
try {
await CF.updateStack(updateInput).promise();
await CF.send(new UpdateStackCommand(updateInput));
} catch (error: any) {
if (error['message'].includes('No updates are to be performed')) {
CloudRunnerLogger.log(`No updates are to be performed`);
@ -93,7 +109,13 @@ export class AWSBaseStack {
);
}
if (stack.StackStatus === 'UPDATE_IN_PROGRESS') {
await CF.waitFor('stackUpdateComplete', describeStackInput).promise();
await waitUntilStackUpdateComplete(
{
client: CF,
maxWaitTime: 200,
},
describeStackInput,
);
}
}
CloudRunnerLogger.log('base stack is now ready');

6
src/model/cloud-runner/providers/aws/aws-error.ts
View File

@ -1,15 +1,15 @@
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import * as SDK from 'aws-sdk';
import { CloudFormation, DescribeStackEventsCommand } from '@aws-sdk/client-cloudformation';
import * as core from '@actions/core';
import CloudRunner from '../../cloud-runner';
export class AWSError {
static async handleStackCreationFailure(error: any, CF: SDK.CloudFormation, taskDefStackName: string) {
static async handleStackCreationFailure(error: any, CF: CloudFormation, taskDefStackName: string) {
CloudRunnerLogger.log('aws error: ');
core.error(JSON.stringify(error, undefined, 4));
if (CloudRunner.buildParameters.cloudRunnerDebug) {
CloudRunnerLogger.log('Getting events and resources for task stack');
const events = (await CF.describeStackEvents({ StackName: taskDefStackName }).promise()).StackEvents;
const events = (await CF.send(new DescribeStackEventsCommand({ StackName: taskDefStackName }))).StackEvents;
CloudRunnerLogger.log(JSON.stringify(events, undefined, 4));
}
}

43
src/model/cloud-runner/providers/aws/aws-job-stack.ts
View File

@ -1,4 +1,12 @@
import * as SDK from 'aws-sdk';
import {
CloudFormation,
CreateStackCommand,
CreateStackCommandInput,
DescribeStackResourcesCommand,
DescribeStacksCommand,
ListStacksCommand,
waitUntilStackCreateComplete,
} from '@aws-sdk/client-cloudformation';
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import { AWSCloudFormationTemplates } from './aws-cloud-formation-templates';
@ -16,7 +24,7 @@ export class AWSJobStack {
}
public async setupCloudFormations(
CF: SDK.CloudFormation,
CF: CloudFormation,
buildGuid: string,
image: string,
entrypoint: string[],
@ -119,7 +127,7 @@ export class AWSJobStack {
let previousStackExists = true;
while (previousStackExists) {
previousStackExists = false;
const stacks = await CF.listStacks().promise();
const stacks = await CF.send(new ListStacksCommand({}));
if (!stacks.StackSummaries) {
throw new Error('Faild to get stacks');
}
@ -132,7 +140,7 @@ export class AWSJobStack {
}
}
}
const createStackInput: SDK.CloudFormation.CreateStackInput = {
const createStackInput: CreateStackCommandInput = {
StackName: taskDefStackName,
TemplateBody: taskDefCloudFormation,
Capabilities: ['CAPABILITY_IAM'],
@ -140,9 +148,15 @@ export class AWSJobStack {
};
try {
CloudRunnerLogger.log(`Creating job aws formation ${taskDefStackName}`);
await CF.createStack(createStackInput).promise();
await CF.waitFor('stackCreateComplete', { StackName: taskDefStackName }).promise();
const describeStack = await CF.describeStacks({ StackName: taskDefStackName }).promise();
await CF.send(new CreateStackCommand(createStackInput));
await waitUntilStackCreateComplete(
{
client: CF,
maxWaitTime: 200,
},
{ StackName: taskDefStackName },
);
const describeStack = await CF.send(new DescribeStacksCommand({ StackName: taskDefStackName }));
for (const parameter of parameters) {
if (!describeStack.Stacks?.[0].Parameters?.some((x) => x.ParameterKey === parameter.ParameterKey)) {
throw new Error(`Parameter ${parameter.ParameterKey} not found in stack`);
@ -153,7 +167,7 @@ export class AWSJobStack {
throw error;
}
const createCleanupStackInput: SDK.CloudFormation.CreateStackInput = {
const createCleanupStackInput: CreateStackCommandInput = {
StackName: `${taskDefStackName}-cleanup`,
TemplateBody: CleanupCronFormation.formation,
Capabilities: ['CAPABILITY_IAM'],
@ -183,7 +197,7 @@ export class AWSJobStack {
if (CloudRunnerOptions.useCleanupCron) {
try {
CloudRunnerLogger.log(`Creating job cleanup formation`);
await CF.createStack(createCleanupStackInput).promise();
await CF.send(new CreateStackCommand(createCleanupStackInput));
// await CF.waitFor('stackCreateComplete', { StackName: createCleanupStackInput.StackName }).promise();
} catch (error) {
@ -193,12 +207,15 @@ export class AWSJobStack {
}
const taskDefResources = (
await CF.describeStackResources({
StackName: taskDefStackName,
}).promise()
await CF.send(
new DescribeStackResourcesCommand({
StackName: taskDefStackName,
}),
)
).StackResources;
const baseResources = (await CF.describeStackResources({ StackName: this.baseStackName }).promise()).StackResources;
const baseResources = (await CF.send(new DescribeStackResourcesCommand({ StackName: this.baseStackName })))
.StackResources;
return {
taskDefStackName,

70
src/model/cloud-runner/providers/aws/aws-task-runner.ts
View File

@ -1,4 +1,19 @@
import * as AWS from 'aws-sdk';
import {
DescribeTasksCommand,
ECS,
RunTaskCommand,
RunTaskCommandInput,
Task,
waitUntilTasksRunning,
} from '@aws-sdk/client-ecs';
import {
DescribeStreamCommand,
DescribeStreamCommandOutput,
GetRecordsCommand,
GetRecordsCommandOutput,
GetShardIteratorCommand,
Kinesis,
} from '@aws-sdk/client-kinesis';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import * as core from '@actions/core';
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
@ -12,8 +27,8 @@ import CloudRunnerOptions from '../../options/cloud-runner-options';
import GitHub from '../../../github';
class AWSTaskRunner {
public static ECS: AWS.ECS;
public static Kinesis: AWS.Kinesis;
public static ECS: ECS;
public static Kinesis: Kinesis;
private static readonly encodedUnderscore = `$252F`;
static async runTask(
taskDef: CloudRunnerAWSTaskDef,
@ -60,7 +75,7 @@ class AWSTaskRunner {
throw new Error(`Container Overrides length must be at most 8192`);
}
const task = await AWSTaskRunner.ECS.runTask(runParameters).promise();
const task = await AWSTaskRunner.ECS.send(new RunTaskCommand(runParameters as RunTaskCommandInput));
const taskArn = task.tasks?.[0].taskArn || '';
CloudRunnerLogger.log('Cloud runner job is starting');
await AWSTaskRunner.waitUntilTaskRunning(taskArn, cluster);
@ -108,7 +123,13 @@ class AWSTaskRunner {
private static async waitUntilTaskRunning(taskArn: string, cluster: string) {
try {
await AWSTaskRunner.ECS.waitFor('tasksRunning', { tasks: [taskArn], cluster }).promise();
await waitUntilTasksRunning(
{
client: AWSTaskRunner.ECS,
maxWaitTime: 120,
},
{ tasks: [taskArn], cluster },
);
} catch (error_) {
const error = error_ as Error;
await new Promise((resolve) => setTimeout(resolve, 3000));
@ -124,10 +145,7 @@ class AWSTaskRunner {
}
static async describeTasks(clusterName: string, taskArn: string) {
const tasks = await AWSTaskRunner.ECS.describeTasks({
cluster: clusterName,
tasks: [taskArn],
}).promise();
const tasks = await AWSTaskRunner.ECS.send(new DescribeTasksCommand({ cluster: clusterName, tasks: [taskArn] }));
if (tasks.tasks?.[0]) {
return tasks.tasks?.[0];
} else {
@ -169,9 +187,7 @@ class AWSTaskRunner {
output: string,
shouldCleanup: boolean,
) {
const records = await AWSTaskRunner.Kinesis.getRecords({
ShardIterator: iterator,
}).promise();
const records = await AWSTaskRunner.Kinesis.send(new GetRecordsCommand({ ShardIterator: iterator }));
iterator = records.NextShardIterator || '';
({ shouldReadLogs, output, shouldCleanup } = AWSTaskRunner.logRecords(
records,
@ -184,7 +200,7 @@ class AWSTaskRunner {
return { iterator, shouldReadLogs, output, shouldCleanup };
}
private static checkStreamingShouldContinue(taskData: AWS.ECS.Task, timestamp: number, shouldReadLogs: boolean) {
private static checkStreamingShouldContinue(taskData: Task, timestamp: number, shouldReadLogs: boolean) {
if (taskData?.lastStatus === 'UNKNOWN') {
CloudRunnerLogger.log('## Cloud runner job unknwon');
}
@ -204,15 +220,17 @@ class AWSTaskRunner {
}
private static logRecords(
records: AWS.Kinesis.GetRecordsOutput,
records: GetRecordsCommandOutput,
iterator: string,
shouldReadLogs: boolean,
output: string,
shouldCleanup: boolean,
) {
if (records.Records.length > 0 && iterator) {
for (const record of records.Records) {
const json = JSON.parse(zlib.gunzipSync(Buffer.from(record.Data as string, 'base64')).toString('utf8'));
if ((records.Records ?? []).length > 0 && iterator) {
for (const record of records.Records ?? []) {
const json = JSON.parse(
zlib.gunzipSync(Buffer.from(record.Data as unknown as string, 'base64')).toString('utf8'),
);
if (json.messageType === 'DATA_MESSAGE') {
for (const logEvent of json.logEvents) {
({ shouldReadLogs, shouldCleanup, output } = FollowLogStreamService.handleIteration(
@ -230,19 +248,19 @@ class AWSTaskRunner {
}
private static async getLogStream(kinesisStreamName: string) {
return await AWSTaskRunner.Kinesis.describeStream({
StreamName: kinesisStreamName,
}).promise();
return await AWSTaskRunner.Kinesis.send(new DescribeStreamCommand({ StreamName: kinesisStreamName }));
}
private static async getLogIterator(stream: AWS.Kinesis.DescribeStreamOutput) {
private static async getLogIterator(stream: DescribeStreamCommandOutput) {
return (
(
await AWSTaskRunner.Kinesis.getShardIterator({
ShardIteratorType: 'TRIM_HORIZON',
StreamName: stream.StreamDescription.StreamName,
ShardId: stream.StreamDescription.Shards[0].ShardId,
}).promise()
await AWSTaskRunner.Kinesis.send(
new GetShardIteratorCommand({
ShardIteratorType: 'TRIM_HORIZON',
StreamName: stream.StreamDescription?.StreamName ?? '',
ShardId: stream.StreamDescription?.Shards?.[0]?.ShardId || '',
}),
)
).ShardIterator || ''
);
}

6
src/model/cloud-runner/providers/aws/cloud-runner-aws-task-def.ts
View File

@ -1,9 +1,9 @@
import * as AWS from 'aws-sdk';
import { StackResource } from '@aws-sdk/client-cloudformation';
class CloudRunnerAWSTaskDef {
public taskDefStackName!: string;
public taskDefCloudFormation!: string;
public taskDefResources: AWS.CloudFormation.StackResources | undefined;
public baseResources: AWS.CloudFormation.StackResources | undefined;
public taskDefResources: Array<StackResource> | undefined;
public baseResources: Array<StackResource> | undefined;
}
export default CloudRunnerAWSTaskDef;

46
src/model/cloud-runner/providers/aws/index.ts
View File

@ -1,4 +1,6 @@
import * as SDK from 'aws-sdk';
import { CloudFormation, DeleteStackCommand, waitUntilStackDeleteComplete } from '@aws-sdk/client-cloudformation';
import { ECS as ECSClient } from '@aws-sdk/client-ecs';
import { Kinesis } from '@aws-sdk/client-kinesis';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import CloudRunnerAWSTaskDef from './cloud-runner-aws-task-def';
@ -75,7 +77,7 @@ class AWSBuildEnvironment implements ProviderInterface {
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
) {
process.env.AWS_REGION = Input.region;
const CF = new SDK.CloudFormation();
const CF = new CloudFormation({ region: Input.region });
await new AwsBaseStack(this.baseStackName).setupBaseStack(CF);
}
@ -89,10 +91,10 @@ class AWSBuildEnvironment implements ProviderInterface {
secrets: CloudRunnerSecret[],
): Promise<string> {
process.env.AWS_REGION = Input.region;
const ECS = new SDK.ECS();
const CF = new SDK.CloudFormation();
const ECS = new ECSClient({ region: Input.region });
const CF = new CloudFormation({ region: Input.region });
AwsTaskRunner.ECS = ECS;
AwsTaskRunner.Kinesis = new SDK.Kinesis();
AwsTaskRunner.Kinesis = new Kinesis({ region: Input.region });
CloudRunnerLogger.log(`AWS Region: ${CF.config.region}`);
const entrypoint = ['/bin/sh'];
const startTimeMs = Date.now();
@ -129,23 +131,31 @@ class AWSBuildEnvironment implements ProviderInterface {
}
}
async cleanupResources(CF: SDK.CloudFormation, taskDef: CloudRunnerAWSTaskDef) {
async cleanupResources(CF: CloudFormation, taskDef: CloudRunnerAWSTaskDef) {
CloudRunnerLogger.log('Cleanup starting');
await CF.deleteStack({
StackName: taskDef.taskDefStackName,
}).promise();
await CF.send(new DeleteStackCommand({ StackName: taskDef.taskDefStackName }));
if (CloudRunnerOptions.useCleanupCron) {
await CF.deleteStack({
StackName: `${taskDef.taskDefStackName}-cleanup`,
}).promise();
await CF.send(new DeleteStackCommand({ StackName: `${taskDef.taskDefStackName}-cleanup` }));
}
await CF.waitFor('stackDeleteComplete', {
StackName: taskDef.taskDefStackName,
}).promise();
await CF.waitFor('stackDeleteComplete', {
StackName: `${taskDef.taskDefStackName}-cleanup`,
}).promise();
await waitUntilStackDeleteComplete(
{
client: CF,
maxWaitTime: 200,
},
{
StackName: taskDef.taskDefStackName,
},
);
await waitUntilStackDeleteComplete(
{
client: CF,
maxWaitTime: 200,
},
{
StackName: `${taskDef.taskDefStackName}-cleanup`,
},
);
CloudRunnerLogger.log(`Deleted Stack: ${taskDef.taskDefStackName}`);
CloudRunnerLogger.log('Cleanup complete');
}