actions/upload-artifact
1
0
Fork 0
mirror of https://github.com/actions/upload-artifact.git synced 2024-11-22 21:35:29 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Update NPM packages

Browse Source
This commit is contained in:
Konrad Pabjan 2020-04-09 21:35:51 +02:00
parent 40fe78158e
commit 85bdb8f4a9
3 changed files with 1383 additions and 731 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

629
dist/index.js vendored
View File

@ -2453,6 +2453,77 @@ if (process.env.NODE_DEBUG && /\btunnel\b/.test(process.env.NODE_DEBUG)) {
exports.debug = debug; // for test exports.debug = debug; // for test
/***/ }),
/***/ 176:
/***/ (function(__unusedmodule, exports, __webpack_require__) {
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const core_1 = __webpack_require__(470);
/**
* Status Reporter that displays information about the progress/status of an artifact that is being uploaded or downloaded
*
* Variable display time that can be adjusted using the displayFrequencyInMilliseconds variable
* The total status of the upload/download gets displayed according to this value
* If there is a large file that is being uploaded, extra information about the individual status can also be displayed using the updateLargeFileStatus function
*/
class StatusReporter {
constructor(displayFrequencyInMilliseconds) {
this.totalNumberOfFilesToProcess = 0;
this.processedCount = 0;
this.largeFiles = new Map();
this.totalFileStatus = undefined;
this.largeFileStatus = undefined;
this.displayFrequencyInMilliseconds = displayFrequencyInMilliseconds;
}
setTotalNumberOfFilesToProcess(fileTotal) {
this.totalNumberOfFilesToProcess = fileTotal;
}
start() {
// displays information about the total upload/download status
this.totalFileStatus = setInterval(() => {
// display 1 decimal place without any rounding
const percentage = this.formatPercentage(this.processedCount, this.totalNumberOfFilesToProcess);
core_1.info(`Total file count: ${this.totalNumberOfFilesToProcess} ---- Processed file #${this.processedCount} (${percentage.slice(0, percentage.indexOf('.') + 2)}%)`);
}, this.displayFrequencyInMilliseconds);
// displays extra information about any large files that take a significant amount of time to upload or download every 1 second
this.largeFileStatus = setInterval(() => {
for (const value of Array.from(this.largeFiles.values())) {
core_1.info(value);
}
// delete all entires in the map after displaying the information so it will not be displayed again unless explicitly added
this.largeFiles.clear();
}, 1000);
}
// if there is a large file that is being uploaded in chunks, this is used to display extra information about the status of the upload
updateLargeFileStatus(fileName, numerator, denominator) {
// display 1 decimal place without any rounding
const percentage = this.formatPercentage(numerator, denominator);
const displayInformation = `Uploading ${fileName} (${percentage.slice(0, percentage.indexOf('.') + 2)}%)`;
// any previously added display information should be overwritten for the specific large file because a map is being used
this.largeFiles.set(fileName, displayInformation);
}
stop() {
if (this.totalFileStatus) {
clearInterval(this.totalFileStatus);
}
if (this.largeFileStatus) {
clearInterval(this.largeFileStatus);
}
}
incrementProcessedCount() {
this.processedCount++;
}
formatPercentage(numerator, denominator) {
// toFixed() rounds, so use extra precision to display accurate information even though 4 decimal places are not displayed
return ((numerator / denominator) * 100).toFixed(4).toString();
}
}
exports.StatusReporter = StatusReporter;
//# sourceMappingURL=status-reporter.js.map
/***/ }), /***/ }),
/***/ 211: /***/ 211:
@ -2480,75 +2551,6 @@ exports.create = create;
/***/ }), /***/ }),
/***/ 221:
/***/ (function(__unusedmodule, exports, __webpack_require__) {
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const core_1 = __webpack_require__(470);
/**
* Upload Status Reporter that displays information about the progress/status of an artifact that is being uploaded
*
* Every 10 seconds, the total status of the upload gets displayed. If there is a large file that is being uploaded,
* extra information about the individual status of an upload can also be displayed
*/
class UploadStatusReporter {
constructor() {
this.totalNumberOfFilesToUpload = 0;
this.processedCount = 0;
this.largeUploads = new Map();
this.totalUploadStatus = undefined;
this.largeFileUploadStatus = undefined;
}
setTotalNumberOfFilesToUpload(fileTotal) {
this.totalNumberOfFilesToUpload = fileTotal;
}
start() {
const _this = this;
// displays information about the total upload status every 10 seconds
this.totalUploadStatus = setInterval(function () {
// display 1 decimal place without any rounding
const percentage = _this.formatPercentage(_this.processedCount, _this.totalNumberOfFilesToUpload);
core_1.info(`Total file(s): ${_this.totalNumberOfFilesToUpload} ---- Processed file #${_this.processedCount} (${percentage.slice(0, percentage.indexOf('.') + 2)}%)`);
}, 10000);
// displays extra information about any large files that take a significant amount of time to upload every 1 second
this.largeFileUploadStatus = setInterval(function () {
for (const value of Array.from(_this.largeUploads.values())) {
core_1.info(value);
}
// delete all entires in the map after displaying the information so it will not be displayed again unless explicitly added
_this.largeUploads = new Map();
}, 1000);
}
updateLargeFileStatus(fileName, numerator, denomiator) {
// display 1 decimal place without any rounding
const percentage = this.formatPercentage(numerator, denomiator);
const displayInformation = `Uploading ${fileName} (${percentage.slice(0, percentage.indexOf('.') + 2)}%)`;
// any previously added display information should be overwritten for the specific large file because a map is being used
this.largeUploads.set(fileName, displayInformation);
}
stop() {
if (this.totalUploadStatus) {
clearInterval(this.totalUploadStatus);
}
if (this.largeFileUploadStatus) {
clearInterval(this.largeFileUploadStatus);
}
}
incrementProcessedCount() {
this.processedCount++;
}
formatPercentage(numerator, denominator) {
// toFixed() rounds, so use extra precision to display accurate information even though 4 decimal places are not displayed
return ((numerator / denominator) * 100).toFixed(4).toString();
}
}
exports.UploadStatusReporter = UploadStatusReporter;
//# sourceMappingURL=upload-status-reporter.js.map
/***/ }),
/***/ 226: /***/ 226:
/***/ (function(__unusedmodule, exports) { /***/ (function(__unusedmodule, exports) {
@ -3809,6 +3811,8 @@ class DefaultArtifactClient {
else { else {
// Create all necessary directories recursively before starting any download // Create all necessary directories recursively before starting any download
yield utils_1.createDirectoriesForArtifact(downloadSpecification.directoryStructure); yield utils_1.createDirectoriesForArtifact(downloadSpecification.directoryStructure);
core.info('Directory structure has been setup for the artifact');
yield utils_1.createEmptyFilesForArtifact(downloadSpecification.emptyFilesToCreate);
yield downloadHttpClient.downloadSingleArtifact(downloadSpecification.filesToDownload); yield downloadHttpClient.downloadSingleArtifact(downloadSpecification.filesToDownload);
} }
return { return {
@ -3843,6 +3847,7 @@ class DefaultArtifactClient {
} }
else { else {
yield utils_1.createDirectoriesForArtifact(downloadSpecification.directoryStructure); yield utils_1.createDirectoriesForArtifact(downloadSpecification.directoryStructure);
yield utils_1.createEmptyFilesForArtifact(downloadSpecification.emptyFilesToCreate);
yield downloadHttpClient.downloadSingleArtifact(downloadSpecification.filesToDownload); yield downloadHttpClient.downloadSingleArtifact(downloadSpecification.filesToDownload);
} }
response.push({ response.push({
@ -4018,22 +4023,34 @@ run();
"use strict"; "use strict";
Object.defineProperty(exports, "__esModule", { value: true }); Object.defineProperty(exports, "__esModule", { value: true });
// The number of concurrent uploads that happens at the same time
function getUploadFileConcurrency() { function getUploadFileConcurrency() {
return 2; return 2;
} }
exports.getUploadFileConcurrency = getUploadFileConcurrency; exports.getUploadFileConcurrency = getUploadFileConcurrency;
// When uploading large files that can't be uploaded with a single http call, this controls
// the chunk size that is used during upload
function getUploadChunkSize() { function getUploadChunkSize() {
return 4 * 1024 * 1024; // 4 MB Chunks return 4 * 1024 * 1024; // 4 MB Chunks
} }
exports.getUploadChunkSize = getUploadChunkSize; exports.getUploadChunkSize = getUploadChunkSize;
function getUploadRetryCount() { // The maximum number of retries that can be attempted before an upload or download fails
return 3; function getRetryLimit() {
return 5;
} }
exports.getUploadRetryCount = getUploadRetryCount; exports.getRetryLimit = getRetryLimit;
function getRetryWaitTimeInMilliseconds() { // With exponential backoff, the larger the retry count, the larger the wait time before another attempt
return 10000; // The retry multiplier controls by how much the backOff time increases depending on the number of retries
function getRetryMultiplier() {
return 1.5;
} }
exports.getRetryWaitTimeInMilliseconds = getRetryWaitTimeInMilliseconds; exports.getRetryMultiplier = getRetryMultiplier;
// The initial wait time if an upload or download fails and a retry is being attempted for the first time
function getInitialRetryIntervalInMilliseconds() {
return 3000;
}
exports.getInitialRetryIntervalInMilliseconds = getInitialRetryIntervalInMilliseconds;
// The number of concurrent downloads that happens at the same time
function getDownloadFileConcurrency() { function getDownloadFileConcurrency() {
return 2; return 2;
} }
@ -5218,12 +5235,14 @@ const path = __importStar(__webpack_require__(622));
* @param includeRootDirectory specifies if there should be an extra directory (denoted by the artifact name) where the artifact files should be downloaded to * @param includeRootDirectory specifies if there should be an extra directory (denoted by the artifact name) where the artifact files should be downloaded to
*/ */
function getDownloadSpecification(artifactName, artifactEntries, downloadPath, includeRootDirectory) { function getDownloadSpecification(artifactName, artifactEntries, downloadPath, includeRootDirectory) {
// use a set for the directory paths so that there are no duplicates
const directories = new Set(); const directories = new Set();
const specifications = { const specifications = {
rootDownloadLocation: includeRootDirectory rootDownloadLocation: includeRootDirectory
? path.join(downloadPath, artifactName) ? path.join(downloadPath, artifactName)
: downloadPath, : downloadPath,
directoryStructure: [], directoryStructure: [],
emptyFilesToCreate: [],
filesToDownload: [] filesToDownload: []
}; };
for (const entry of artifactEntries) { for (const entry of artifactEntries) {
@ -5241,6 +5260,11 @@ function getDownloadSpecification(artifactName, artifactEntries, downloadPath, i
if (entry.itemType === 'file') { if (entry.itemType === 'file') {
// Get the directories that we need to create from the filePath for each individual file // Get the directories that we need to create from the filePath for each individual file
directories.add(path.dirname(filePath)); directories.add(path.dirname(filePath));
if (entry.fileLength === 0) {
// An empty file was uploaded, create the empty files locally so that no extra http calls are made
specifications.emptyFilesToCreate.push(filePath);
}
else {
specifications.filesToDownload.push({ specifications.filesToDownload.push({
sourceLocation: entry.contentLocation, sourceLocation: entry.contentLocation,
targetPath: filePath targetPath: filePath
@ -5248,6 +5272,7 @@ function getDownloadSpecification(artifactName, artifactEntries, downloadPath, i
} }
} }
} }
}
specifications.directoryStructure = Array.from(directories); specifications.directoryStructure = Array.from(directories);
return specifications; return specifications;
} }
@ -5290,6 +5315,7 @@ var HttpCodes;
HttpCodes[HttpCodes["RequestTimeout"] = 408] = "RequestTimeout"; HttpCodes[HttpCodes["RequestTimeout"] = 408] = "RequestTimeout";
HttpCodes[HttpCodes["Conflict"] = 409] = "Conflict"; HttpCodes[HttpCodes["Conflict"] = 409] = "Conflict";
HttpCodes[HttpCodes["Gone"] = 410] = "Gone"; HttpCodes[HttpCodes["Gone"] = 410] = "Gone";
HttpCodes[HttpCodes["TooManyRequests"] = 429] = "TooManyRequests";
HttpCodes[HttpCodes["InternalServerError"] = 500] = "InternalServerError"; HttpCodes[HttpCodes["InternalServerError"] = 500] = "InternalServerError";
HttpCodes[HttpCodes["NotImplemented"] = 501] = "NotImplemented"; HttpCodes[HttpCodes["NotImplemented"] = 501] = "NotImplemented";
HttpCodes[HttpCodes["BadGateway"] = 502] = "BadGateway"; HttpCodes[HttpCodes["BadGateway"] = 502] = "BadGateway";
@ -6458,6 +6484,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
}; };
Object.defineProperty(exports, "__esModule", { value: true }); Object.defineProperty(exports, "__esModule", { value: true });
const fs = __importStar(__webpack_require__(747)); const fs = __importStar(__webpack_require__(747));
const core = __importStar(__webpack_require__(470));
const tmp = __importStar(__webpack_require__(875)); const tmp = __importStar(__webpack_require__(875));
const stream = __importStar(__webpack_require__(794)); const stream = __importStar(__webpack_require__(794));
const utils_1 = __webpack_require__(870); const utils_1 = __webpack_require__(870);
@ -6465,15 +6492,14 @@ const config_variables_1 = __webpack_require__(401);
const util_1 = __webpack_require__(669); const util_1 = __webpack_require__(669);
const url_1 = __webpack_require__(835); const url_1 = __webpack_require__(835);
const perf_hooks_1 = __webpack_require__(630); const perf_hooks_1 = __webpack_require__(630);
const upload_status_reporter_1 = __webpack_require__(221); const status_reporter_1 = __webpack_require__(176);
const core_1 = __webpack_require__(470);
const http_manager_1 = __webpack_require__(452); const http_manager_1 = __webpack_require__(452);
const upload_gzip_1 = __webpack_require__(647); const upload_gzip_1 = __webpack_require__(647);
const stat = util_1.promisify(fs.stat); const stat = util_1.promisify(fs.stat);
class UploadHttpClient { class UploadHttpClient {
constructor() { constructor() {
this.uploadHttpManager = new http_manager_1.HttpManager(config_variables_1.getUploadFileConcurrency()); this.uploadHttpManager = new http_manager_1.HttpManager(config_variables_1.getUploadFileConcurrency());
this.statusReporter = new upload_status_reporter_1.UploadStatusReporter(); this.statusReporter = new status_reporter_1.StatusReporter(10000);
} }
/** /**
* Creates a file container for the new artifact in the remote blob storage/file service * Creates a file container for the new artifact in the remote blob storage/file service
@ -6488,18 +6514,22 @@ class UploadHttpClient {
}; };
const data = JSON.stringify(parameters, null, 2); const data = JSON.stringify(parameters, null, 2);
const artifactUrl = utils_1.getArtifactUrl(); const artifactUrl = utils_1.getArtifactUrl();
// use the first client from the httpManager, `keep-alive` is not used so the connection will close immediatly // use the first client from the httpManager, `keep-alive` is not used so the connection will close immediately
const client = this.uploadHttpManager.getClient(0); const client = this.uploadHttpManager.getClient(0);
const requestOptions = utils_1.getRequestOptions('application/json', false, false); const requestOptions = utils_1.getUploadRequestOptions('application/json', false);
const rawResponse = yield client.post(artifactUrl, data, requestOptions); const rawResponse = yield client.post(artifactUrl, data, requestOptions);
const body = yield rawResponse.readBody(); const body = yield rawResponse.readBody();
if (utils_1.isSuccessStatusCode(rawResponse.message.statusCode) && body) { if (utils_1.isSuccessStatusCode(rawResponse.message.statusCode) && body) {
return JSON.parse(body); return JSON.parse(body);
} }
else if (utils_1.isForbiddenStatusCode(rawResponse.message.statusCode)) {
// if a 403 is returned when trying to create a file container, the customer has exceeded
// their storage quota so no new artifact containers can be created
throw new Error(`Artifact storage quota has been hit. Unable to upload any new artifacts`);
}
else { else {
// eslint-disable-next-line no-console utils_1.displayHttpDiagnostics(rawResponse);
console.log(rawResponse); throw new Error(`Unable to create a container for the artifact ${artifactName} at ${artifactUrl}`);
throw new Error(`Unable to create a container for the artifact ${artifactName}`);
} }
}); });
} }
@ -6513,7 +6543,7 @@ class UploadHttpClient {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
const FILE_CONCURRENCY = config_variables_1.getUploadFileConcurrency(); const FILE_CONCURRENCY = config_variables_1.getUploadFileConcurrency();
const MAX_CHUNK_SIZE = config_variables_1.getUploadChunkSize(); const MAX_CHUNK_SIZE = config_variables_1.getUploadChunkSize();
core_1.debug(`File Concurrency: ${FILE_CONCURRENCY}, and Chunk Size: ${MAX_CHUNK_SIZE}`); core.debug(`File Concurrency: ${FILE_CONCURRENCY}, and Chunk Size: ${MAX_CHUNK_SIZE}`);
const parameters = []; const parameters = [];
// by default, file uploads will continue if there is an error unless specified differently in the options // by default, file uploads will continue if there is an error unless specified differently in the options
let continueOnError = true; let continueOnError = true;
@ -6540,7 +6570,7 @@ class UploadHttpClient {
let uploadFileSize = 0; let uploadFileSize = 0;
let totalFileSize = 0; let totalFileSize = 0;
let abortPendingFileUploads = false; let abortPendingFileUploads = false;
this.statusReporter.setTotalNumberOfFilesToUpload(filesToUpload.length); this.statusReporter.setTotalNumberOfFilesToProcess(filesToUpload.length);
this.statusReporter.start(); this.statusReporter.start();
// only allow a certain amount of files to be uploaded at once, this is done to reduce potential errors // only allow a certain amount of files to be uploaded at once, this is done to reduce potential errors
yield Promise.all(parallelUploads.map((index) => __awaiter(this, void 0, void 0, function* () { yield Promise.all(parallelUploads.map((index) => __awaiter(this, void 0, void 0, function* () {
@ -6553,13 +6583,16 @@ class UploadHttpClient {
} }
const startTime = perf_hooks_1.performance.now(); const startTime = perf_hooks_1.performance.now();
const uploadFileResult = yield this.uploadFileAsync(index, currentFileParameters); const uploadFileResult = yield this.uploadFileAsync(index, currentFileParameters);
core_1.debug(`File: ${++completedFiles}/${filesToUpload.length}. ${currentFileParameters.file} took ${(perf_hooks_1.performance.now() - startTime).toFixed(3)} milliseconds to finish upload`); if (core.isDebug()) {
uploadFileSize += uploadFileResult.successfullUploadSize; core.debug(`File: ${++completedFiles}/${filesToUpload.length}. ${currentFileParameters.file} took ${(perf_hooks_1.performance.now() - startTime).toFixed(3)} milliseconds to finish upload`);
}
uploadFileSize += uploadFileResult.successfulUploadSize;
totalFileSize += uploadFileResult.totalSize; totalFileSize += uploadFileResult.totalSize;
if (uploadFileResult.isSuccess === false) { if (uploadFileResult.isSuccess === false) {
failedItemsToReport.push(currentFileParameters.file); failedItemsToReport.push(currentFileParameters.file);
if (!continueOnError) { if (!continueOnError) {
// existing uploads will be able to finish however all pending uploads will fail fast // fail fast
core.error(`aborting artifact upload`);
abortPendingFileUploads = true; abortPendingFileUploads = true;
} }
} }
@ -6569,7 +6602,7 @@ class UploadHttpClient {
this.statusReporter.stop(); this.statusReporter.stop();
// done uploading, safety dispose all connections // done uploading, safety dispose all connections
this.uploadHttpManager.disposeAndReplaceAllClients(); this.uploadHttpManager.disposeAndReplaceAllClients();
core_1.info(`Total size of all the files uploaded is ${uploadFileSize} bytes`); core.info(`Total size of all the files uploaded is ${uploadFileSize} bytes`);
return { return {
uploadSize: uploadFileSize, uploadSize: uploadFileSize,
totalSize: totalFileSize, totalSize: totalFileSize,
@ -6592,7 +6625,7 @@ class UploadHttpClient {
let failedChunkSizes = 0; let failedChunkSizes = 0;
let uploadFileSize = 0; let uploadFileSize = 0;
let isGzip = true; let isGzip = true;
// the file that is being uploaded is less than 64k in size, to increase thoroughput and to minimize disk I/O // the file that is being uploaded is less than 64k in size, to increase throughput and to minimize disk I/O
// for creating a new GZip file, an in-memory buffer is used for compression // for creating a new GZip file, an in-memory buffer is used for compression
if (totalFileSize < 65536) { if (totalFileSize < 65536) {
const buffer = yield upload_gzip_1.createGZipFileInBuffer(parameters.file); const buffer = yield upload_gzip_1.createGZipFileInBuffer(parameters.file);
@ -6615,16 +6648,16 @@ class UploadHttpClient {
// chunk failed to upload // chunk failed to upload
isUploadSuccessful = false; isUploadSuccessful = false;
failedChunkSizes += uploadFileSize; failedChunkSizes += uploadFileSize;
core_1.warning(`Aborting upload for ${parameters.file} due to failure`); core.warning(`Aborting upload for ${parameters.file} due to failure`);
} }
return { return {
isSuccess: isUploadSuccessful, isSuccess: isUploadSuccessful,
successfullUploadSize: uploadFileSize - failedChunkSizes, successfulUploadSize: uploadFileSize - failedChunkSizes,
totalSize: totalFileSize totalSize: totalFileSize
}; };
} }
else { else {
// the file that is being uploaded is greater than 64k in size, a temprorary file gets created on disk using the // the file that is being uploaded is greater than 64k in size, a temporary file gets created on disk using the
// npm tmp-promise package and this file gets used during compression for the GZip file that gets created // npm tmp-promise package and this file gets used during compression for the GZip file that gets created
return tmp return tmp
.file() .file()
@ -6643,11 +6676,6 @@ class UploadHttpClient {
// upload only a single chunk at a time // upload only a single chunk at a time
while (offset < uploadFileSize) { while (offset < uploadFileSize) {
const chunkSize = Math.min(uploadFileSize - offset, parameters.maxChunkSize); const chunkSize = Math.min(uploadFileSize - offset, parameters.maxChunkSize);
if (abortFileUpload) {
// if we don't want to continue in the event of an error, any pending upload chunks will be marked as failed
failedChunkSizes += chunkSize;
continue;
}
// if an individual file is greater than 100MB (1024*1024*100) in size, display extra information about the upload status // if an individual file is greater than 100MB (1024*1024*100) in size, display extra information about the upload status
if (uploadFileSize > 104857600) { if (uploadFileSize > 104857600) {
this.statusReporter.updateLargeFileStatus(parameters.file, offset, uploadFileSize); this.statusReporter.updateLargeFileStatus(parameters.file, offset, uploadFileSize);
@ -6655,6 +6683,11 @@ class UploadHttpClient {
const start = offset; const start = offset;
const end = offset + chunkSize - 1; const end = offset + chunkSize - 1;
offset += parameters.maxChunkSize; offset += parameters.maxChunkSize;
if (abortFileUpload) {
// if we don't want to continue in the event of an error, any pending upload chunks will be marked as failed
failedChunkSizes += chunkSize;
continue;
}
const result = yield this.uploadChunk(httpClientIndex, parameters.resourceUrl, fs.createReadStream(uploadFilePath, { const result = yield this.uploadChunk(httpClientIndex, parameters.resourceUrl, fs.createReadStream(uploadFilePath, {
start, start,
end, end,
@ -6665,7 +6698,7 @@ class UploadHttpClient {
// successfully uploaded so the server may report a different size for what was uploaded // successfully uploaded so the server may report a different size for what was uploaded
isUploadSuccessful = false; isUploadSuccessful = false;
failedChunkSizes += chunkSize; failedChunkSizes += chunkSize;
core_1.warning(`Aborting upload for ${parameters.file} due to failure`); core.warning(`Aborting upload for ${parameters.file} due to failure`);
abortFileUpload = true; abortFileUpload = true;
} }
} }
@ -6675,7 +6708,7 @@ class UploadHttpClient {
return new Promise(resolve => { return new Promise(resolve => {
resolve({ resolve({
isSuccess: isUploadSuccessful, isSuccess: isUploadSuccessful,
successfullUploadSize: uploadFileSize - failedChunkSizes, successfulUploadSize: uploadFileSize - failedChunkSizes,
totalSize: totalFileSize totalSize: totalFileSize
}); });
}); });
@ -6699,17 +6732,57 @@ class UploadHttpClient {
uploadChunk(httpClientIndex, resourceUrl, data, start, end, uploadFileSize, isGzip, totalFileSize) { uploadChunk(httpClientIndex, resourceUrl, data, start, end, uploadFileSize, isGzip, totalFileSize) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
// prepare all the necessary headers before making any http call // prepare all the necessary headers before making any http call
const requestOptions = utils_1.getRequestOptions('application/octet-stream', true, isGzip, totalFileSize, end - start + 1, utils_1.getContentRange(start, end, uploadFileSize)); const requestOptions = utils_1.getUploadRequestOptions('application/octet-stream', true, isGzip, totalFileSize, end - start + 1, utils_1.getContentRange(start, end, uploadFileSize));
const uploadChunkRequest = () => __awaiter(this, void 0, void 0, function* () { const uploadChunkRequest = () => __awaiter(this, void 0, void 0, function* () {
const client = this.uploadHttpManager.getClient(httpClientIndex); const client = this.uploadHttpManager.getClient(httpClientIndex);
return yield client.sendStream('PUT', resourceUrl, data, requestOptions); return yield client.sendStream('PUT', resourceUrl, data, requestOptions);
}); });
let retryCount = 0; let retryCount = 0;
const retryLimit = config_variables_1.getUploadRetryCount(); const retryLimit = config_variables_1.getRetryLimit();
// Increments the current retry count and then checks if the retry limit has been reached
// If there have been too many retries, fail so the download stops
const incrementAndCheckRetryLimit = (response) => {
retryCount++;
if (retryCount > retryLimit) {
if (response) {
utils_1.displayHttpDiagnostics(response);
}
core.info(`Retry limit has been reached for chunk at offset ${start} to ${resourceUrl}`);
return true;
}
return false;
};
const backOff = (retryAfterValue) => __awaiter(this, void 0, void 0, function* () {
this.uploadHttpManager.disposeAndReplaceClient(httpClientIndex);
if (retryAfterValue) {
core.info(`Backoff due to too many requests, retry #${retryCount}. Waiting for ${retryAfterValue} milliseconds before continuing the upload`);
yield new Promise(resolve => setTimeout(resolve, retryAfterValue));
}
else {
const backoffTime = utils_1.getExponentialRetryTimeInMilliseconds(retryCount);
core.info(`Exponential backoff for retry #${retryCount}. Waiting for ${backoffTime} milliseconds before continuing the upload at offset ${start}`);
yield new Promise(resolve => setTimeout(resolve, backoffTime));
}
core.info(`Finished backoff for retry #${retryCount}, continuing with upload`);
return;
});
// allow for failed chunks to be retried multiple times // allow for failed chunks to be retried multiple times
while (retryCount <= retryLimit) { while (retryCount <= retryLimit) {
let response;
try { try {
const response = yield uploadChunkRequest(); response = yield uploadChunkRequest();
}
catch (error) {
// if an error is caught, it is usually indicative of a timeout so retry the upload
core.info(`An error has been caught http-client index ${httpClientIndex}, retrying the upload`);
// eslint-disable-next-line no-console
console.log(error);
if (incrementAndCheckRetryLimit()) {
return false;
}
yield backOff();
continue;
}
// Always read the body of the response. There is potential for a resource leak if the body is not read which will // Always read the body of the response. There is potential for a resource leak if the body is not read which will
// result in the connection remaining open along with unintended consequences when trying to dispose of the client // result in the connection remaining open along with unintended consequences when trying to dispose of the client
yield response.readBody(); yield response.readBody();
@ -6717,39 +6790,20 @@ class UploadHttpClient {
return true; return true;
} }
else if (utils_1.isRetryableStatusCode(response.message.statusCode)) { else if (utils_1.isRetryableStatusCode(response.message.statusCode)) {
retryCount++; core.info(`A ${response.message.statusCode} status code has been received, will attempt to retry the upload`);
if (retryCount > retryLimit) { if (incrementAndCheckRetryLimit(response)) {
core_1.info(`Retry limit has been reached for chunk at offset ${start} to ${resourceUrl}`);
return false; return false;
} }
else { utils_1.isThrottledStatusCode(response.message.statusCode)
core_1.info(`HTTP ${response.message.statusCode} during chunk upload, will retry at offset ${start} after ${config_variables_1.getRetryWaitTimeInMilliseconds} milliseconds. Retry count #${retryCount}. URL ${resourceUrl}`); ? yield backOff(utils_1.tryGetRetryAfterValueTimeInMilliseconds(response.message.headers))
this.uploadHttpManager.disposeAndReplaceClient(httpClientIndex); : yield backOff();
yield new Promise(resolve => setTimeout(resolve, config_variables_1.getRetryWaitTimeInMilliseconds()));
}
} }
else { else {
core_1.info(`#ERROR# Unable to upload chunk to ${resourceUrl}`); core.error(`Unexpected response. Unable to upload chunk to ${resourceUrl}`);
// eslint-disable-next-line no-console utils_1.displayHttpDiagnostics(response);
console.log(response);
return false; return false;
} }
} }
catch (error) {
// eslint-disable-next-line no-console
console.log(error);
retryCount++;
if (retryCount > retryLimit) {
core_1.info(`Retry limit has been reached for chunk at offset ${start} to ${resourceUrl}`);
return false;
}
else {
core_1.info(`Retrying chunk upload after encountering an error`);
this.uploadHttpManager.disposeAndReplaceClient(httpClientIndex);
yield new Promise(resolve => setTimeout(resolve, config_variables_1.getRetryWaitTimeInMilliseconds()));
}
}
}
return false; return false;
}); });
} }
@ -6759,26 +6813,26 @@ class UploadHttpClient {
*/ */
patchArtifactSize(size, artifactName) { patchArtifactSize(size, artifactName) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
const requestOptions = utils_1.getRequestOptions('application/json', false, false); const requestOptions = utils_1.getUploadRequestOptions('application/json', false);
const resourceUrl = new url_1.URL(utils_1.getArtifactUrl()); const resourceUrl = new url_1.URL(utils_1.getArtifactUrl());
resourceUrl.searchParams.append('artifactName', artifactName); resourceUrl.searchParams.append('artifactName', artifactName);
const parameters = { Size: size }; const parameters = { Size: size };
const data = JSON.stringify(parameters, null, 2); const data = JSON.stringify(parameters, null, 2);
core_1.debug(`URL is ${resourceUrl.toString()}`); core.debug(`URL is ${resourceUrl.toString()}`);
// use the first client from the httpManager, `keep-alive` is not used so the connection will close immediatly // use the first client from the httpManager, `keep-alive` is not used so the connection will close immediately
const client = this.uploadHttpManager.getClient(0); const client = this.uploadHttpManager.getClient(0);
const rawResponse = yield client.patch(resourceUrl.toString(), data, requestOptions); const response = yield client.patch(resourceUrl.toString(), data, requestOptions);
const body = yield rawResponse.readBody(); const body = yield response.readBody();
if (utils_1.isSuccessStatusCode(rawResponse.message.statusCode)) { if (utils_1.isSuccessStatusCode(response.message.statusCode)) {
core_1.debug(`Artifact ${artifactName} has been successfully uploaded, total size ${size}`); core.debug(`Artifact ${artifactName} has been successfully uploaded, total size in bytes: ${size}`);
} }
else if (rawResponse.message.statusCode === 404) { else if (response.message.statusCode === 404) {
throw new Error(`An Artifact with the name ${artifactName} was not found`); throw new Error(`An Artifact with the name ${artifactName} was not found`);
} }
else { else {
// eslint-disable-next-line no-console utils_1.displayHttpDiagnostics(response);
console.log(body); core.info(body);
throw new Error(`Unable to finish uploading artifact ${artifactName}`); throw new Error(`Unable to finish uploading artifact ${artifactName} to ${resourceUrl}`);
} }
}); });
} }
@ -6955,7 +7009,7 @@ function createGZipFileInBuffer(originalFilePath) {
const inputStream = fs.createReadStream(originalFilePath); const inputStream = fs.createReadStream(originalFilePath);
const gzip = zlib.createGzip(); const gzip = zlib.createGzip();
inputStream.pipe(gzip); inputStream.pipe(gzip);
// read stream into buffer, using experimental async itterators see https://github.com/nodejs/readable-stream/issues/403#issuecomment-479069043 // read stream into buffer, using experimental async iterators see https://github.com/nodejs/readable-stream/issues/403#issuecomment-479069043
const chunks = []; const chunks = [];
try { try {
for (var gzip_1 = __asyncValues(gzip), gzip_1_1; gzip_1_1 = yield gzip_1.next(), !gzip_1_1.done;) { for (var gzip_1 = __asyncValues(gzip), gzip_1_1; gzip_1_1 = yield gzip_1.next(), !gzip_1_1.done;) {
@ -7178,15 +7232,19 @@ var __importStar = (this && this.__importStar) || function (mod) {
}; };
Object.defineProperty(exports, "__esModule", { value: true }); Object.defineProperty(exports, "__esModule", { value: true });
const fs = __importStar(__webpack_require__(747)); const fs = __importStar(__webpack_require__(747));
const core = __importStar(__webpack_require__(470));
const zlib = __importStar(__webpack_require__(761)); const zlib = __importStar(__webpack_require__(761));
const utils_1 = __webpack_require__(870); const utils_1 = __webpack_require__(870);
const url_1 = __webpack_require__(835); const url_1 = __webpack_require__(835);
const status_reporter_1 = __webpack_require__(176);
const perf_hooks_1 = __webpack_require__(630);
const http_manager_1 = __webpack_require__(452); const http_manager_1 = __webpack_require__(452);
const config_variables_1 = __webpack_require__(401); const config_variables_1 = __webpack_require__(401);
const core_1 = __webpack_require__(470);
class DownloadHttpClient { class DownloadHttpClient {
constructor() { constructor() {
this.downloadHttpManager = new http_manager_1.HttpManager(config_variables_1.getDownloadFileConcurrency()); this.downloadHttpManager = new http_manager_1.HttpManager(config_variables_1.getDownloadFileConcurrency());
// downloads are usually significantly faster than uploads so display status information every second
this.statusReporter = new status_reporter_1.StatusReporter(1000);
} }
/** /**
* Gets a list of all artifacts that are in a specific container * Gets a list of all artifacts that are in a specific container
@ -7194,17 +7252,16 @@ class DownloadHttpClient {
listArtifacts() { listArtifacts() {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
const artifactUrl = utils_1.getArtifactUrl(); const artifactUrl = utils_1.getArtifactUrl();
// use the first client from the httpManager, `keep-alive` is not used so the connection will close immediatly // use the first client from the httpManager, `keep-alive` is not used so the connection will close immediately
const client = this.downloadHttpManager.getClient(0); const client = this.downloadHttpManager.getClient(0);
const requestOptions = utils_1.getRequestOptions('application/json'); const requestOptions = utils_1.getDownloadRequestOptions('application/json');
const rawResponse = yield client.get(artifactUrl, requestOptions); const response = yield client.get(artifactUrl, requestOptions);
const body = yield rawResponse.readBody(); const body = yield response.readBody();
if (utils_1.isSuccessStatusCode(rawResponse.message.statusCode) && body) { if (utils_1.isSuccessStatusCode(response.message.statusCode) && body) {
return JSON.parse(body); return JSON.parse(body);
} }
// eslint-disable-next-line no-console utils_1.displayHttpDiagnostics(response);
console.log(rawResponse); throw new Error(`Unable to list artifacts for the run. Resource Url ${artifactUrl}`);
throw new Error(`Unable to list artifacts for the run`);
}); });
} }
/** /**
@ -7217,17 +7274,15 @@ class DownloadHttpClient {
// the itemPath search parameter controls which containers will be returned // the itemPath search parameter controls which containers will be returned
const resourceUrl = new url_1.URL(containerUrl); const resourceUrl = new url_1.URL(containerUrl);
resourceUrl.searchParams.append('itemPath', artifactName); resourceUrl.searchParams.append('itemPath', artifactName);
// no concurrent calls so a single httpClient without the http-manager is sufficient // use the first client from the httpManager, `keep-alive` is not used so the connection will close immediately
const client = utils_1.createHttpClient(); const client = this.downloadHttpManager.getClient(0);
// no keep-alive header, client disposal is not necessary const requestOptions = utils_1.getDownloadRequestOptions('application/json');
const requestOptions = utils_1.getRequestOptions('application/json'); const response = yield client.get(resourceUrl.toString(), requestOptions);
const rawResponse = yield client.get(resourceUrl.toString(), requestOptions); const body = yield response.readBody();
const body = yield rawResponse.readBody(); if (utils_1.isSuccessStatusCode(response.message.statusCode) && body) {
if (utils_1.isSuccessStatusCode(rawResponse.message.statusCode) && body) {
return JSON.parse(body); return JSON.parse(body);
} }
// eslint-disable-next-line no-console utils_1.displayHttpDiagnostics(response);
console.log(rawResponse);
throw new Error(`Unable to get ContainersItems from ${resourceUrl}`); throw new Error(`Unable to get ContainersItems from ${resourceUrl}`);
}); });
} }
@ -7239,18 +7294,34 @@ class DownloadHttpClient {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
const DOWNLOAD_CONCURRENCY = config_variables_1.getDownloadFileConcurrency(); const DOWNLOAD_CONCURRENCY = config_variables_1.getDownloadFileConcurrency();
// limit the number of files downloaded at a single time // limit the number of files downloaded at a single time
core.debug(`Download file concurrency is set to ${DOWNLOAD_CONCURRENCY}`);
const parallelDownloads = [...new Array(DOWNLOAD_CONCURRENCY).keys()]; const parallelDownloads = [...new Array(DOWNLOAD_CONCURRENCY).keys()];
let currentFile = 0;
let downloadedFiles = 0; let downloadedFiles = 0;
core.info(`Total number of files that will be downloaded: ${downloadItems.length}`);
this.statusReporter.setTotalNumberOfFilesToProcess(downloadItems.length);
this.statusReporter.start();
yield Promise.all(parallelDownloads.map((index) => __awaiter(this, void 0, void 0, function* () { yield Promise.all(parallelDownloads.map((index) => __awaiter(this, void 0, void 0, function* () {
while (downloadedFiles < downloadItems.length) { while (currentFile < downloadItems.length) {
const currentFileToDownload = downloadItems[downloadedFiles]; const currentFileToDownload = downloadItems[currentFile];
downloadedFiles += 1; currentFile += 1;
const startTime = perf_hooks_1.performance.now();
yield this.downloadIndividualFile(index, currentFileToDownload.sourceLocation, currentFileToDownload.targetPath); yield this.downloadIndividualFile(index, currentFileToDownload.sourceLocation, currentFileToDownload.targetPath);
if (core.isDebug()) {
core.debug(`File: ${++downloadedFiles}/${downloadItems.length}. ${currentFileToDownload.targetPath} took ${(perf_hooks_1.performance.now() - startTime).toFixed(3)} milliseconds to finish downloading`);
} }
}))); this.statusReporter.incrementProcessedCount();
// done downloading, safety dispose all connections }
})))
.catch(error => {
throw new Error(`Unable to download the artifact: ${error}`);
})
.finally(() => {
this.statusReporter.stop();
// safety dispose all connections
this.downloadHttpManager.disposeAndReplaceAllClients(); this.downloadHttpManager.disposeAndReplaceAllClients();
}); });
});
} }
/** /**
* Downloads an individual file * Downloads an individual file
@ -7260,64 +7331,114 @@ class DownloadHttpClient {
*/ */
downloadIndividualFile(httpClientIndex, artifactLocation, downloadPath) { downloadIndividualFile(httpClientIndex, artifactLocation, downloadPath) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
const stream = fs.createWriteStream(downloadPath); let retryCount = 0;
const retryLimit = config_variables_1.getRetryLimit();
const destinationStream = fs.createWriteStream(downloadPath);
const requestOptions = utils_1.getDownloadRequestOptions('application/json', true, true);
// a single GET request is used to download a file
const makeDownloadRequest = () => __awaiter(this, void 0, void 0, function* () {
const client = this.downloadHttpManager.getClient(httpClientIndex); const client = this.downloadHttpManager.getClient(httpClientIndex);
const requestOptions = utils_1.getRequestOptions('application/octet-stream', true); return yield client.get(artifactLocation, requestOptions);
const response = yield client.get(artifactLocation, requestOptions); });
// check the response headers to determine if the file was compressed using gzip // check the response headers to determine if the file was compressed using gzip
const isGzip = (headers) => { const isGzip = (headers) => {
return ('content-encoding' in headers && headers['content-encoding'] === 'gzip'); return ('content-encoding' in headers && headers['content-encoding'] === 'gzip');
}; };
// Increments the current retry count and then checks if the retry limit has been reached
// If there have been too many retries, fail so the download stops. If there is a retryAfterValue value provided,
// it will be used
const backOff = (retryAfterValue) => __awaiter(this, void 0, void 0, function* () {
retryCount++;
if (retryCount > retryLimit) {
return Promise.reject(new Error(`Retry limit has been reached. Unable to download ${artifactLocation}`));
}
else {
this.downloadHttpManager.disposeAndReplaceClient(httpClientIndex);
if (retryAfterValue) {
// Back off by waiting the specified time denoted by the retry-after header
core.info(`Backoff due to too many requests, retry #${retryCount}. Waiting for ${retryAfterValue} milliseconds before continuing the download`);
yield new Promise(resolve => setTimeout(resolve, retryAfterValue));
}
else {
// Back off using an exponential value that depends on the retry count
const backoffTime = utils_1.getExponentialRetryTimeInMilliseconds(retryCount);
core.info(`Exponential backoff for retry #${retryCount}. Waiting for ${backoffTime} milliseconds before continuing the download`);
yield new Promise(resolve => setTimeout(resolve, backoffTime));
}
core.info(`Finished backoff for retry #${retryCount}, continuing with download`);
}
});
// keep trying to download a file until a retry limit has been reached
while (retryCount <= retryLimit) {
let response;
try {
response = yield makeDownloadRequest();
}
catch (error) {
// if an error is caught, it is usually indicative of a timeout so retry the download
core.info('An error occurred while attempting to download a file');
// eslint-disable-next-line no-console
console.log(error);
// increment the retryCount and use exponential backoff to wait before making the next request
yield backOff();
continue;
}
if (utils_1.isSuccessStatusCode(response.message.statusCode)) { if (utils_1.isSuccessStatusCode(response.message.statusCode)) {
yield this.pipeResponseToStream(response, stream, isGzip(response.message.headers)); // The body contains the contents of the file however calling response.readBody() causes all the content to be converted to a string
// which can cause some gzip encoded data to be lost
// Instead of using response.readBody(), response.message is a readableStream that can be directly used to get the raw body contents
return this.pipeResponseToFile(response, destinationStream, isGzip(response.message.headers));
} }
else if (utils_1.isRetryableStatusCode(response.message.statusCode)) { else if (utils_1.isRetryableStatusCode(response.message.statusCode)) {
core_1.warning(`Received http ${response.message.statusCode} during file download, will retry ${artifactLocation} after 10 seconds`); core.info(`A ${response.message.statusCode} response code has been received while attempting to download an artifact`);
// if an error is encountered, dispose of the http connection, and create a new one // if a throttled status code is received, try to get the retryAfter header value, else differ to standard exponential backoff
this.downloadHttpManager.disposeAndReplaceClient(httpClientIndex); utils_1.isThrottledStatusCode(response.message.statusCode)
yield new Promise(resolve => setTimeout(resolve, config_variables_1.getRetryWaitTimeInMilliseconds())); ? yield backOff(utils_1.tryGetRetryAfterValueTimeInMilliseconds(response.message.headers))
const retryResponse = yield client.get(artifactLocation); : yield backOff();
if (utils_1.isSuccessStatusCode(retryResponse.message.statusCode)) {
yield this.pipeResponseToStream(response, stream, isGzip(response.message.headers));
} }
else { else {
// eslint-disable-next-line no-console // Some unexpected response code, fail immediately and stop the download
console.log(retryResponse); utils_1.displayHttpDiagnostics(response);
throw new Error(`Unable to download ${artifactLocation}`); return Promise.reject(new Error(`Unexpected http ${response.message.statusCode} during download for ${artifactLocation}`));
} }
} }
else {
// eslint-disable-next-line no-console
console.log(response);
throw new Error(`Unable to download ${artifactLocation}`);
}
}); });
} }
/** /**
* Pipes the response from downloading an individual file to the appropriate stream * Pipes the response from downloading an individual file to the appropriate destination stream while decoding gzip content if necessary
* @param response the http response recieved when downloading a file * @param response the http response received when downloading a file
* @param stream the stream where the file should be written to * @param destinationStream the stream where the file should be written to
* @param isGzip does the response need to be be uncompressed * @param isGzip a boolean denoting if the content is compressed using gzip and if we need to decode it
*/ */
pipeResponseToStream(response, stream, isGzip) { pipeResponseToFile(response, destinationStream, isGzip) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
return new Promise(resolve => { yield new Promise((resolve, reject) => {
if (isGzip) { if (isGzip) {
// pipe the response into gunzip to decompress
const gunzip = zlib.createGunzip(); const gunzip = zlib.createGunzip();
response.message response.message
.pipe(gunzip) .pipe(gunzip)
.pipe(stream) .pipe(destinationStream)
.on('close', () => { .on('close', () => {
resolve(); resolve();
})
.on('error', error => {
core.error(`An error has been encountered while decompressing and writing a downloaded file to ${destinationStream.path}`);
reject(error);
}); });
} }
else { else {
response.message.pipe(stream).on('close', () => { response.message
.pipe(destinationStream)
.on('close', () => {
resolve(); resolve();
})
.on('error', error => {
core.error(`An error has been encountered while writing a downloaded file to ${destinationStream.path}`);
reject(error);
}); });
} }
}); });
return;
}); });
} }
} }
@ -7593,6 +7714,23 @@ const fs_1 = __webpack_require__(747);
const http_client_1 = __webpack_require__(539); const http_client_1 = __webpack_require__(539);
const auth_1 = __webpack_require__(226); const auth_1 = __webpack_require__(226);
const config_variables_1 = __webpack_require__(401); const config_variables_1 = __webpack_require__(401);
/**
* Returns a retry time in milliseconds that exponentially gets larger
* depending on the amount of retries that have been attempted
*/
function getExponentialRetryTimeInMilliseconds(retryCount) {
if (retryCount < 0) {
throw new Error('RetryCount should not be negative');
}
else if (retryCount === 0) {
return config_variables_1.getInitialRetryIntervalInMilliseconds();
}
const minTime = config_variables_1.getInitialRetryIntervalInMilliseconds() * config_variables_1.getRetryMultiplier() * retryCount;
const maxTime = minTime * config_variables_1.getRetryMultiplier();
// returns a random number between the minTime (inclusive) and the maxTime (exclusive)
return Math.random() * (maxTime - minTime) + minTime;
}
exports.getExponentialRetryTimeInMilliseconds = getExponentialRetryTimeInMilliseconds;
/** /**
* Parses a env variable that is a number * Parses a env variable that is a number
*/ */
@ -7618,6 +7756,13 @@ function isSuccessStatusCode(statusCode) {
return statusCode >= 200 && statusCode < 300; return statusCode >= 200 && statusCode < 300;
} }
exports.isSuccessStatusCode = isSuccessStatusCode; exports.isSuccessStatusCode = isSuccessStatusCode;
function isForbiddenStatusCode(statusCode) {
if (!statusCode) {
return false;
}
return statusCode === http_client_1.HttpCodes.Forbidden;
}
exports.isForbiddenStatusCode = isForbiddenStatusCode;
function isRetryableStatusCode(statusCode) { function isRetryableStatusCode(statusCode) {
if (!statusCode) { if (!statusCode) {
return false; return false;
@ -7625,11 +7770,40 @@ function isRetryableStatusCode(statusCode) {
const retryableStatusCodes = [ const retryableStatusCodes = [
http_client_1.HttpCodes.BadGateway, http_client_1.HttpCodes.BadGateway,
http_client_1.HttpCodes.ServiceUnavailable, http_client_1.HttpCodes.ServiceUnavailable,
http_client_1.HttpCodes.GatewayTimeout http_client_1.HttpCodes.GatewayTimeout,
http_client_1.HttpCodes.TooManyRequests
]; ];
return retryableStatusCodes.includes(statusCode); return retryableStatusCodes.includes(statusCode);
} }
exports.isRetryableStatusCode = isRetryableStatusCode; exports.isRetryableStatusCode = isRetryableStatusCode;
function isThrottledStatusCode(statusCode) {
if (!statusCode) {
return false;
}
return statusCode === http_client_1.HttpCodes.TooManyRequests;
}
exports.isThrottledStatusCode = isThrottledStatusCode;
/**
* Attempts to get the retry-after value from a set of http headers. The retry time
* is originally denoted in seconds, so if present, it is converted to milliseconds
* @param headers all the headers received when making an http call
*/
function tryGetRetryAfterValueTimeInMilliseconds(headers) {
if (headers['retry-after']) {
const retryTime = Number(headers['retry-after']);
if (!isNaN(retryTime)) {
core_1.info(`Retry-After header is present with a value of ${retryTime}`);
return retryTime * 1000;
}
core_1.info(`Returned retry-after header value: ${retryTime} is non-numeric and cannot be used`);
return undefined;
}
core_1.info(`No retry-after header was found. Dumping all headers for diagnostic purposes`);
// eslint-disable-next-line no-console
console.log(headers);
return undefined;
}
exports.tryGetRetryAfterValueTimeInMilliseconds = tryGetRetryAfterValueTimeInMilliseconds;
function getContentRange(start, end, total) { function getContentRange(start, end, total) {
// Format: `bytes start-end/fileSize // Format: `bytes start-end/fileSize
// start and end are inclusive // start and end are inclusive
@ -7639,20 +7813,48 @@ function getContentRange(start, end, total) {
} }
exports.getContentRange = getContentRange; exports.getContentRange = getContentRange;
/** /**
* Sets all the necessary headers when making HTTP calls * Sets all the necessary headers when downloading an artifact
* @param {string} contentType the type of content being uploaded
* @param {boolean} isKeepAlive is the same connection being used to make multiple calls
* @param {boolean} acceptGzip can we accept a gzip encoded response
* @param {string} acceptType the type of content that we can accept
* @returns appropriate request options to make a specific http call during artifact download
*/
function getDownloadRequestOptions(contentType, isKeepAlive, acceptGzip) {
const requestOptions = {};
if (contentType) {
requestOptions['Content-Type'] = contentType;
}
if (isKeepAlive) {
requestOptions['Connection'] = 'Keep-Alive';
// keep alive for at least 10 seconds before closing the connection
requestOptions['Keep-Alive'] = '10';
}
if (acceptGzip) {
// if we are expecting a response with gzip encoding, it should be using an octet-stream in the accept header
requestOptions['Accept-Encoding'] = 'gzip';
requestOptions['Accept'] = `application/octet-stream;api-version=${getApiVersion()}`;
}
else {
// default to application/json if we are not working with gzip content
requestOptions['Accept'] = `application/json;api-version=${getApiVersion()}`;
}
return requestOptions;
}
exports.getDownloadRequestOptions = getDownloadRequestOptions;
/**
* Sets all the necessary headers when uploading an artifact
* @param {string} contentType the type of content being uploaded * @param {string} contentType the type of content being uploaded
* @param {boolean} isKeepAlive is the same connection being used to make multiple calls * @param {boolean} isKeepAlive is the same connection being used to make multiple calls
* @param {boolean} isGzip is the connection being used to upload GZip compressed content * @param {boolean} isGzip is the connection being used to upload GZip compressed content
* @param {number} uncompressedLength the original size of the content if something is being uploaded that has been compressed * @param {number} uncompressedLength the original size of the content if something is being uploaded that has been compressed
* @param {number} contentLength the length of the content that is being uploaded * @param {number} contentLength the length of the content that is being uploaded
* @param {string} contentRange the range of the content that is being uploaded * @param {string} contentRange the range of the content that is being uploaded
* @returns appropriate request options to make a specific http call * @returns appropriate request options to make a specific http call during artifact upload
*/ */
function getRequestOptions(contentType, isKeepAlive, isGzip, uncompressedLength, contentLength, contentRange) { function getUploadRequestOptions(contentType, isKeepAlive, isGzip, uncompressedLength, contentLength, contentRange) {
const requestOptions = { const requestOptions = {};
// same Accept type for each http call that gets made requestOptions['Accept'] = `application/json;api-version=${getApiVersion()}`;
Accept: `application/json;api-version=${getApiVersion()}`
};
if (contentType) { if (contentType) {
requestOptions['Content-Type'] = contentType; requestOptions['Content-Type'] = contentType;
} }
@ -7673,7 +7875,7 @@ function getRequestOptions(contentType, isKeepAlive, isGzip, uncompressedLength,
} }
return requestOptions; return requestOptions;
} }
exports.getRequestOptions = getRequestOptions; exports.getUploadRequestOptions = getUploadRequestOptions;
function createHttpClient() { function createHttpClient() {
return new http_client_1.HttpClient('action/artifact', [ return new http_client_1.HttpClient('action/artifact', [
new auth_1.BearerCredentialHandler(config_variables_1.getRuntimeToken()) new auth_1.BearerCredentialHandler(config_variables_1.getRuntimeToken())
@ -7686,6 +7888,23 @@ function getArtifactUrl() {
return artifactUrl; return artifactUrl;
} }
exports.getArtifactUrl = getArtifactUrl; exports.getArtifactUrl = getArtifactUrl;
/**
* Uh oh! Something might have gone wrong during either upload or download. The IHtttpClientResponse object contains information
* about the http call that was made by the actions http client. This information might be useful to display for diagnostic purposes, but
* this entire object is really big and most of the information is not really useful. This function takes the response object and displays only
* the information that we want.
*
* Certain information such as the TLSSocket and the Readable state are not really useful for diagnostic purposes so they can be avoided.
* Other information such as the headers, the response code and message might be useful, so this is displayed.
*/
function displayHttpDiagnostics(response) {
core_1.info(`##### Begin Diagnostic HTTP information #####
Status Code: ${response.message.statusCode}
Status Message: ${response.message.statusMessage}
Header Information: ${JSON.stringify(response.message.headers, undefined, 2)}
###### End Diagnostic HTTP information ######`);
}
exports.displayHttpDiagnostics = displayHttpDiagnostics;
/** /**
* Invalid characters that cannot be in the artifact name or an uploaded file. Will be rejected * Invalid characters that cannot be in the artifact name or an uploaded file. Will be rejected
* from the server if attempted to be sent over. These characters are not allowed due to limitations with certain * from the server if attempted to be sent over. These characters are not allowed due to limitations with certain
@ -7747,6 +7966,14 @@ function createDirectoriesForArtifact(directories) {
}); });
} }
exports.createDirectoriesForArtifact = createDirectoriesForArtifact; exports.createDirectoriesForArtifact = createDirectoriesForArtifact;
function createEmptyFilesForArtifact(emptyFilesToCreate) {
return __awaiter(this, void 0, void 0, function* () {
for (const filePath of emptyFilesToCreate) {
yield (yield fs_1.promises.open(filePath, 'w')).close();
}
});
}
exports.createEmptyFilesForArtifact = createEmptyFilesForArtifact;
//# sourceMappingURL=utils.js.map //# sourceMappingURL=utils.js.map
/***/ }), /***/ }),

1439
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

18
package.json
View File

@ -29,23 +29,23 @@
}, },
"homepage": "https://github.com/actions/upload-artifact#readme", "homepage": "https://github.com/actions/upload-artifact#readme",
"devDependencies": { "devDependencies": {
"@actions/artifact": "^0.2.0", "@actions/artifact": "^0.3.0",
"@actions/core": "^1.2.3", "@actions/core": "^1.2.3",
"@actions/glob": "^0.1.0", "@actions/glob": "^0.1.0",
"@actions/io": "^1.0.2", "@actions/io": "^1.0.2",
"@types/jest": "^25.1.4", "@types/jest": "^25.2.1",
"@types/node": "^12.12.30", "@types/node": "^13.11.1",
"@typescript-eslint/parser": "^2.23.0", "@typescript-eslint/parser": "^2.27.0",
"@zeit/ncc": "^0.20.5", "@zeit/ncc": "^0.22.1",
"concurrently": "^5.1.0", "concurrently": "^5.1.0",
"eslint": "^6.8.0", "eslint": "^6.8.0",
"eslint-plugin-github": "^3.4.1", "eslint-plugin-github": "^3.4.1",
"eslint-plugin-jest": "^23.8.2", "eslint-plugin-jest": "^23.8.2",
"glob": "^7.1.6", "glob": "^7.1.6",
"jest": "^25.1.0", "jest": "^25.3.0",
"jest-circus": "^25.1.0", "jest-circus": "^25.3.0",
"prettier": "^1.19.1", "prettier": "^2.0.4",
"ts-jest": "^25.2.1", "ts-jest": "^25.3.1",
"typescript": "^3.8.3" "typescript": "^3.8.3"
} }
} }