unity-builder/src/model/cloud-runner/providers/docker/index.ts
Frostebite e73b48fb38
Cloud runner develop - Stabilizes kubernetes provider (#531)
* fixes

* fixes

* fixes

* fixes

* fixes

* check for startup message in workflows

* check for startup message in workflows

* check for startup message in workflows

* check for startup message in workflows

* check for startup message in workflows

* check for startup message in workflows

* Update cloud-runner-ci-pipeline.yml

* Update cloud-runner-ci-pipeline.yml

* no storage class specified

* log file path

* log file path

* log file path

* log file path

* log file path

* log file path

* log file path

* log file path

* updates

* log file path

* latest develop

* log file path

* log file path

* Update package.json

* log file path

* log file path

* log file path

* log file path

* log file path

* log file path

* log file path

* log file path

* log file path

* log file path

* log file path

* log file path

* log file path

* log file path

* stream logs through standard input and new remote client cli command

* stream logs through standard input and new remote client cli command

* stream logs through standard input and new remote client cli command

* stream logs through standard input and new remote client cli command

* stream logs through standard input and new remote client cli command

* stream logs through standard input and new remote client cli command

* stream logs through standard input and new remote client cli command

* stream logs through standard input and new remote client cli command

* stream logs through standard input and new remote client cli command

* stream logs through standard input and new remote client cli command

* stream logs through standard input and new remote client cli command

* stream logs through standard input and new remote client cli command

* stream logs through standard input and new remote client cli command

* stream logs through standard input and new remote client cli command

* stream logs through standard input and new remote client cli command

* update pipeline to use k3s

* version: 'latest'

* fixes

* disable aws pipe for now

* disable aws pipe for now

* disable aws pipe for now

* disable aws pipe for now

* disable aws pipe for now

* disable aws pipe for now

* disable aws pipe for now

* disable aws pipe for now

* disable aws pipe for now

* disable aws pipe for now

* push k8s logs to LOG SERVICE IP

* push k8s logs to LOG SERVICE IP

* push k8s logs to LOG SERVICE IP

* push k8s logs to LOG SERVICE IP

* push k8s logs to LOG SERVICE IP

* push k8s logs to LOG SERVICE IP

* push k8s logs to LOG SERVICE IP

* push k8s logs to LOG SERVICE IP

* tests

* tests

* tests

* tests

* tests

* tests

* tests

* tests

* tests

* tests

* tests

* tests

* tests

* tests

* tests

* tests

* tests

* podname logs for log service

* podname logs for log service

* podname logs for log service

* podname logs for log service

* podname logs for log service

* podname logs for log service

* podname logs for log service

* podname logs for log service

* podname logs for log service

* hashed logs

* hashed logs

* hashed logs

* hashed logs

* hashed logs

* hashed logs

* no wait, just repeat logs

* no wait, just repeat logs

* remove typo - double await

* test fix - kubernetes - name typo in github yaml

* test fix - kubernetes - name typo in github yaml

* check missing log file

* check missing log file

* Push to steam test

* Push to steam test

* Fix path

* k8s reliable log hashing

* k8s reliable log hashing

* k8s reliable log hashing

* hashed logging k8s

* hashed logging k8s

* hashed logging k8s

* hashed logging k8s

* hashed logging k8s

* hashed logging k8s

* Include log chunk when task runner sees log update, clarify if we can pull logs from same line or next line

* Include log chunk when task runner sees log update, clarify if we can pull logs from same line or next line

* Include log chunk when task runner sees log update, clarify if we can pull logs from same line or next line

* Include log chunk when task runner sees log update, clarify if we can pull logs from same line or next line

* Include log chunk when task runner sees log update, clarify if we can pull logs from same line or next line

* Fix exit flow for k8s job

* hash comparison logging for log complete in k8s flow

* Interrupt k8s logs when logs found

* cleanup async parameter

* cleanup async parameter

* cleanup async parameter

* fixes

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix
2024-02-06 23:46:31 +00:00

164 lines
5.7 KiB
TypeScript

import BuildParameters from '../../../build-parameters';
import CloudRunnerEnvironmentVariable from '../../options/cloud-runner-environment-variable';
import CloudRunnerLogger from '../../services/core/cloud-runner-logger';
import { ProviderInterface } from '../provider-interface';
import CloudRunnerSecret from '../../options/cloud-runner-secret';
import Docker from '../../../docker';
import { Action } from '../../..';
import { writeFileSync } from 'node:fs';
import CloudRunner from '../../cloud-runner';
import { ProviderResource } from '../provider-resource';
import { ProviderWorkflow } from '../provider-workflow';
import { CloudRunnerSystem } from '../../services/core/cloud-runner-system';
import * as fs from 'node:fs';
import { CommandHookService } from '../../services/hooks/command-hook-service';
import { StringKeyValuePair } from '../../../shared-types';
class LocalDockerCloudRunner implements ProviderInterface {
public buildParameters!: BuildParameters;
listResources(): Promise<ProviderResource[]> {
return new Promise((resolve) => resolve([]));
}
listWorkflow(): Promise<ProviderWorkflow[]> {
throw new Error('Method not implemented.');
}
watchWorkflow(): Promise<string> {
throw new Error('Method not implemented.');
}
garbageCollect(
// eslint-disable-next-line no-unused-vars
filter: string,
// eslint-disable-next-line no-unused-vars
previewOnly: boolean,
// eslint-disable-next-line no-unused-vars
olderThan: Number,
// eslint-disable-next-line no-unused-vars
fullCache: boolean,
// eslint-disable-next-line no-unused-vars
baseDependencies: boolean,
): Promise<string> {
return new Promise((result) => result(``));
}
async cleanupWorkflow(
buildParameters: BuildParameters,
// eslint-disable-next-line no-unused-vars
branchName: string,
// eslint-disable-next-line no-unused-vars
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
) {
const { workspace } = Action;
if (
fs.existsSync(
`${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
}`,
)
) {
await CloudRunnerSystem.Run(`ls ${workspace}/cloud-runner-cache/cache/build/`);
await CloudRunnerSystem.Run(
`rm -r ${workspace}/cloud-runner-cache/cache/build/build-${buildParameters.buildGuid}.tar${
CloudRunner.buildParameters.useCompressionStrategy ? '.lz4' : ''
}`,
);
}
}
setupWorkflow(
buildGuid: string,
buildParameters: BuildParameters,
// eslint-disable-next-line no-unused-vars
branchName: string,
// eslint-disable-next-line no-unused-vars
defaultSecretsArray: { ParameterKey: string; EnvironmentVariable: string; ParameterValue: string }[],
) {
this.buildParameters = buildParameters;
}
public async runTaskInWorkflow(
buildGuid: string,
image: string,
commands: string,
mountdir: string,
workingdir: string,
environment: CloudRunnerEnvironmentVariable[],
secrets: CloudRunnerSecret[],
): Promise<string> {
CloudRunnerLogger.log(buildGuid);
CloudRunnerLogger.log(commands);
const { workspace, actionFolder } = Action;
const content: StringKeyValuePair[] = [];
for (const x of secrets) {
content.push({ name: x.EnvironmentVariable, value: x.ParameterValue });
}
for (const x of environment) {
content.push({ name: x.name, value: x.value });
}
// if (this.buildParameters?.cloudRunnerIntegrationTests) {
// core.info(JSON.stringify(content, undefined, 4));
// core.info(JSON.stringify(secrets, undefined, 4));
// core.info(JSON.stringify(environment, undefined, 4));
// }
// eslint-disable-next-line unicorn/no-for-loop
for (let index = 0; index < content.length; index++) {
if (content[index] === undefined) {
delete content[index];
}
}
let myOutput = '';
const sharedFolder = `/data/`;
// core.info(JSON.stringify({ workspace, actionFolder, ...this.buildParameters, ...content }, undefined, 4));
const entrypointFilePath = `start.sh`;
const fileContents = `#!/bin/bash
set -e
mkdir -p /github/workspace/cloud-runner-cache
mkdir -p /data/cache
cp -a /github/workspace/cloud-runner-cache/. ${sharedFolder}
${CommandHookService.ApplyHooksToCommands(commands, this.buildParameters)}
cp -a ${sharedFolder}. /github/workspace/cloud-runner-cache/
`;
writeFileSync(`${workspace}/${entrypointFilePath}`, fileContents, {
flag: 'w',
});
if (CloudRunner.buildParameters.cloudRunnerDebug) {
CloudRunnerLogger.log(`Running local-docker: \n ${fileContents}`);
}
if (fs.existsSync(`${workspace}/cloud-runner-cache`)) {
await CloudRunnerSystem.Run(`ls ${workspace}/cloud-runner-cache && du -sh ${workspace}/cloud-runner-cache`);
}
const exitCode = await Docker.run(
image,
{ workspace, actionFolder, ...this.buildParameters },
false,
`chmod +x /github/workspace/${entrypointFilePath} && /github/workspace/${entrypointFilePath}`,
content,
{
listeners: {
stdout: (data: Buffer) => {
myOutput += data.toString();
},
stderr: (data: Buffer) => {
myOutput += `[LOCAL-DOCKER-ERROR]${data.toString()}`;
},
},
},
true,
);
// Docker doesn't exit on fail now so adding this to ensure behavior is unchanged
// TODO: Is there a helpful way to consume the exit code or is it best to except
if (exitCode !== 0) {
throw new Error(`Build failed with exit code ${exitCode}`);
}
return myOutput;
}
}
export default LocalDockerCloudRunner;