mirror of
https://github.com/renovatebot/renovate.git
synced 2024-12-22 13:38:32 +00:00
534 lines
16 KiB
TypeScript
534 lines
16 KiB
TypeScript
import is from '@sindresorhus/is';
|
|
import { DateTime } from 'luxon';
|
|
import {
|
|
PLATFORM_BAD_CREDENTIALS,
|
|
PLATFORM_INTEGRATION_UNAUTHORIZED,
|
|
PLATFORM_RATE_LIMIT_EXCEEDED,
|
|
REPOSITORY_CHANGED,
|
|
} from '../../constants/error-messages';
|
|
import { logger } from '../../logger';
|
|
import { ExternalHostError } from '../../types/errors/external-host-error';
|
|
import { getCache } from '../cache/repository';
|
|
import { maskToken } from '../mask';
|
|
import * as p from '../promises';
|
|
import { range } from '../range';
|
|
import { regEx } from '../regex';
|
|
import { joinUrlParts, parseLinkHeader, resolveBaseUrl } from '../url';
|
|
import { findMatchingRule } from './host-rules';
|
|
import type { GotLegacyError } from './legacy';
|
|
import type {
|
|
GraphqlOptions,
|
|
HttpOptions,
|
|
HttpResponse,
|
|
InternalHttpOptions,
|
|
} from './types';
|
|
import { Http } from '.';
|
|
|
|
const githubBaseUrl = 'https://api.github.com/';
|
|
let baseUrl = githubBaseUrl;
|
|
export const setBaseUrl = (url: string): void => {
|
|
baseUrl = url;
|
|
};
|
|
|
|
export interface GithubHttpOptions extends HttpOptions {
|
|
paginate?: boolean | string;
|
|
paginationField?: string;
|
|
pageLimit?: number;
|
|
repository?: string;
|
|
}
|
|
|
|
interface GithubGraphqlRepoData<T = unknown> {
|
|
repository?: T;
|
|
}
|
|
|
|
export type GithubGraphqlResponse<T = unknown> =
|
|
| {
|
|
data: T;
|
|
errors?: never;
|
|
}
|
|
| {
|
|
data?: never;
|
|
errors: {
|
|
type?: string;
|
|
message: string;
|
|
}[];
|
|
};
|
|
|
|
function handleGotError(
|
|
err: GotLegacyError,
|
|
url: string | URL,
|
|
opts: GithubHttpOptions,
|
|
): Error {
|
|
const path = url.toString();
|
|
let message = err.message || '';
|
|
const body = err.response?.body;
|
|
if (is.plainObject(body) && 'message' in body) {
|
|
message = String(body.message);
|
|
}
|
|
if (
|
|
err.code === 'ERR_HTTP2_STREAM_ERROR' ||
|
|
err.code === 'ENOTFOUND' ||
|
|
err.code === 'ETIMEDOUT' ||
|
|
err.code === 'EAI_AGAIN' ||
|
|
err.code === 'ECONNRESET'
|
|
) {
|
|
logger.debug({ err }, 'GitHub failure: RequestError');
|
|
return new ExternalHostError(err, 'github');
|
|
}
|
|
if (err.name === 'ParseError') {
|
|
logger.debug({ err }, '');
|
|
return new ExternalHostError(err, 'github');
|
|
}
|
|
if (err.statusCode && err.statusCode >= 500 && err.statusCode < 600) {
|
|
logger.debug({ err }, 'GitHub failure: 5xx');
|
|
return new ExternalHostError(err, 'github');
|
|
}
|
|
if (
|
|
err.statusCode === 403 &&
|
|
message.startsWith('You have triggered an abuse detection mechanism')
|
|
) {
|
|
logger.debug({ err }, 'GitHub failure: abuse detection');
|
|
return new Error(PLATFORM_RATE_LIMIT_EXCEEDED);
|
|
}
|
|
if (
|
|
err.statusCode === 403 &&
|
|
message.startsWith('You have exceeded a secondary rate limit')
|
|
) {
|
|
logger.warn({ err }, 'GitHub failure: secondary rate limit');
|
|
return new Error(PLATFORM_RATE_LIMIT_EXCEEDED);
|
|
}
|
|
if (err.statusCode === 403 && message.includes('Upgrade to GitHub Pro')) {
|
|
logger.debug(`Endpoint: ${path}, needs paid GitHub plan`);
|
|
return err;
|
|
}
|
|
if (err.statusCode === 403 && message.includes('rate limit exceeded')) {
|
|
logger.debug({ err }, 'GitHub failure: rate limit');
|
|
return new Error(PLATFORM_RATE_LIMIT_EXCEEDED);
|
|
}
|
|
if (
|
|
err.statusCode === 403 &&
|
|
message.startsWith('Resource not accessible by integration')
|
|
) {
|
|
logger.debug(
|
|
{ err },
|
|
'GitHub failure: Resource not accessible by integration',
|
|
);
|
|
return new Error(PLATFORM_INTEGRATION_UNAUTHORIZED);
|
|
}
|
|
if (err.statusCode === 401 && message.includes('Bad credentials')) {
|
|
const rateLimit = err.headers?.['x-ratelimit-limit'] ?? -1;
|
|
logger.debug(
|
|
{
|
|
token: maskToken(opts.token),
|
|
err,
|
|
},
|
|
'GitHub failure: Bad credentials',
|
|
);
|
|
if (rateLimit === '60') {
|
|
return new ExternalHostError(err, 'github');
|
|
}
|
|
return new Error(PLATFORM_BAD_CREDENTIALS);
|
|
}
|
|
if (err.statusCode === 422) {
|
|
if (
|
|
message.includes('Review cannot be requested from pull request author')
|
|
) {
|
|
return err;
|
|
} else if (err.body?.errors?.find((e: any) => e.field === 'milestone')) {
|
|
return err;
|
|
} else if (err.body?.errors?.find((e: any) => e.code === 'invalid')) {
|
|
logger.debug({ err }, 'Received invalid response - aborting');
|
|
return new Error(REPOSITORY_CHANGED);
|
|
} else if (
|
|
err.body?.errors?.find((e: any) =>
|
|
e.message?.startsWith('A pull request already exists'),
|
|
)
|
|
) {
|
|
return err;
|
|
}
|
|
logger.debug({ err }, '422 Error thrown from GitHub');
|
|
return new ExternalHostError(err, 'github');
|
|
}
|
|
if (
|
|
err.statusCode === 410 &&
|
|
err.body?.message === 'Issues are disabled for this repo'
|
|
) {
|
|
return err;
|
|
}
|
|
return err;
|
|
}
|
|
|
|
interface GraphqlPaginatedContent<T = unknown> {
|
|
nodes: T[];
|
|
edges: T[];
|
|
pageInfo: { hasNextPage: boolean; endCursor: string };
|
|
}
|
|
|
|
function constructAcceptString(input?: any): string {
|
|
const defaultAccept = 'application/vnd.github.v3+json';
|
|
const acceptStrings =
|
|
typeof input === 'string' ? input.split(regEx(/\s*,\s*/)) : [];
|
|
|
|
// TODO: regression of #6736
|
|
if (
|
|
!acceptStrings.some((x) => x === defaultAccept) &&
|
|
(!acceptStrings.some((x) => x.startsWith('application/vnd.github.')) ||
|
|
acceptStrings.length < 2)
|
|
) {
|
|
acceptStrings.push(defaultAccept);
|
|
}
|
|
return acceptStrings.join(', ');
|
|
}
|
|
|
|
const MAX_GRAPHQL_PAGE_SIZE = 100;
|
|
|
|
interface GraphqlPageCacheItem {
|
|
pageLastResizedAt: string;
|
|
pageSize: number;
|
|
}
|
|
|
|
export type GraphqlPageCache = Record<string, GraphqlPageCacheItem>;
|
|
|
|
function getGraphqlPageSize(
|
|
fieldName: string,
|
|
defaultPageSize = MAX_GRAPHQL_PAGE_SIZE,
|
|
): number {
|
|
const cache = getCache();
|
|
const graphqlPageCache = cache?.platform?.github
|
|
?.graphqlPageCache as GraphqlPageCache;
|
|
const cachedRecord = graphqlPageCache?.[fieldName];
|
|
|
|
if (graphqlPageCache && cachedRecord) {
|
|
logger.debug(
|
|
{ fieldName, ...cachedRecord },
|
|
'GraphQL page size: found cached value',
|
|
);
|
|
|
|
const oldPageSize = cachedRecord.pageSize;
|
|
|
|
const now = DateTime.local();
|
|
const then = DateTime.fromISO(cachedRecord.pageLastResizedAt);
|
|
const expiry = then.plus({ hours: 24 });
|
|
if (now > expiry) {
|
|
const newPageSize = Math.min(oldPageSize * 2, MAX_GRAPHQL_PAGE_SIZE);
|
|
if (newPageSize < MAX_GRAPHQL_PAGE_SIZE) {
|
|
const timestamp = now.toISO();
|
|
|
|
logger.debug(
|
|
{ fieldName, oldPageSize, newPageSize, timestamp },
|
|
'GraphQL page size: expanding',
|
|
);
|
|
|
|
cachedRecord.pageLastResizedAt = timestamp;
|
|
cachedRecord.pageSize = newPageSize;
|
|
} else {
|
|
logger.debug(
|
|
{ fieldName, oldPageSize, newPageSize },
|
|
'GraphQL page size: expanded to default page size',
|
|
);
|
|
|
|
delete graphqlPageCache[fieldName];
|
|
}
|
|
|
|
return newPageSize;
|
|
}
|
|
|
|
return oldPageSize;
|
|
}
|
|
|
|
return defaultPageSize;
|
|
}
|
|
|
|
function setGraphqlPageSize(fieldName: string, newPageSize: number): void {
|
|
const oldPageSize = getGraphqlPageSize(fieldName);
|
|
if (newPageSize !== oldPageSize) {
|
|
const now = DateTime.local();
|
|
const pageLastResizedAt = now.toISO();
|
|
logger.debug(
|
|
{ fieldName, oldPageSize, newPageSize, timestamp: pageLastResizedAt },
|
|
'GraphQL page size: shrinking',
|
|
);
|
|
const cache = getCache();
|
|
cache.platform ??= {};
|
|
cache.platform.github ??= {};
|
|
cache.platform.github.graphqlPageCache ??= {};
|
|
const graphqlPageCache = cache.platform.github
|
|
.graphqlPageCache as GraphqlPageCache;
|
|
graphqlPageCache[fieldName] = {
|
|
pageLastResizedAt,
|
|
pageSize: newPageSize,
|
|
};
|
|
}
|
|
}
|
|
|
|
function replaceUrlBase(url: URL, baseUrl: string): URL {
|
|
const relativeUrl = `${url.pathname}${url.search}`;
|
|
return new URL(relativeUrl, baseUrl);
|
|
}
|
|
|
|
export class GithubHttp extends Http<GithubHttpOptions> {
|
|
constructor(hostType = 'github', options?: GithubHttpOptions) {
|
|
super(hostType, options);
|
|
}
|
|
|
|
protected override async request<T>(
|
|
url: string | URL,
|
|
options?: InternalHttpOptions & GithubHttpOptions,
|
|
okToRetry = true,
|
|
): Promise<HttpResponse<T>> {
|
|
const opts: InternalHttpOptions & GithubHttpOptions = {
|
|
baseUrl,
|
|
...options,
|
|
throwHttpErrors: true,
|
|
};
|
|
|
|
if (!opts.token) {
|
|
const authUrl = new URL(resolveBaseUrl(opts.baseUrl!, url));
|
|
|
|
if (opts.repository) {
|
|
// set authUrl to https://api.github.com/repos/org/repo or https://gihub.domain.com/api/v3/repos/org/repo
|
|
authUrl.hash = '';
|
|
authUrl.search = '';
|
|
authUrl.pathname = joinUrlParts(
|
|
authUrl.pathname.startsWith('/api/v3') ? '/api/v3' : '',
|
|
'repos',
|
|
`${opts.repository}`,
|
|
);
|
|
}
|
|
|
|
let readOnly = opts.readOnly;
|
|
const { method = 'get' } = opts;
|
|
if (
|
|
readOnly === undefined &&
|
|
['get', 'head'].includes(method.toLowerCase())
|
|
) {
|
|
readOnly = true;
|
|
}
|
|
const { token } = findMatchingRule(authUrl.toString(), {
|
|
hostType: this.hostType,
|
|
readOnly,
|
|
});
|
|
opts.token = token;
|
|
}
|
|
|
|
const accept = constructAcceptString(opts.headers?.accept);
|
|
|
|
opts.headers = {
|
|
...opts.headers,
|
|
accept,
|
|
};
|
|
|
|
try {
|
|
const result = await super.request<T>(url, opts);
|
|
if (opts.paginate) {
|
|
// Check if result is paginated
|
|
const pageLimit = opts.pageLimit ?? 10;
|
|
const linkHeader = parseLinkHeader(result?.headers?.link);
|
|
const next = linkHeader?.next;
|
|
if (next?.url && linkHeader?.last?.page) {
|
|
let lastPage = parseInt(linkHeader.last.page, 10);
|
|
// istanbul ignore else: needs a test
|
|
if (!process.env.RENOVATE_PAGINATE_ALL && opts.paginate !== 'all') {
|
|
lastPage = Math.min(pageLimit, lastPage);
|
|
}
|
|
const baseUrl = opts.baseUrl;
|
|
const parsedUrl = new URL(next.url, baseUrl);
|
|
const rebasePagination =
|
|
!!baseUrl &&
|
|
!!process.env.RENOVATE_X_REBASE_PAGINATION_LINKS &&
|
|
// Preserve github.com URLs for use cases like release notes
|
|
parsedUrl.origin !== 'https://api.github.com';
|
|
const firstPageUrl = rebasePagination
|
|
? replaceUrlBase(parsedUrl, baseUrl)
|
|
: parsedUrl;
|
|
const queue = [...range(2, lastPage)].map(
|
|
(pageNumber) => (): Promise<HttpResponse<T>> => {
|
|
// copy before modifying searchParams
|
|
const nextUrl = new URL(firstPageUrl);
|
|
nextUrl.searchParams.set('page', String(pageNumber));
|
|
return this.request<T>(
|
|
nextUrl,
|
|
{ ...opts, paginate: false, cacheProvider: undefined },
|
|
okToRetry,
|
|
);
|
|
},
|
|
);
|
|
const pages = await p.all(queue);
|
|
if (opts.paginationField && is.plainObject(result.body)) {
|
|
const paginatedResult = result.body[opts.paginationField];
|
|
if (is.array<T>(paginatedResult)) {
|
|
for (const nextPage of pages) {
|
|
if (is.plainObject(nextPage.body)) {
|
|
const nextPageResults = nextPage.body[opts.paginationField];
|
|
if (is.array<T>(nextPageResults)) {
|
|
paginatedResult.push(...nextPageResults);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} else if (is.array<T>(result.body)) {
|
|
for (const nextPage of pages) {
|
|
if (is.array<T>(nextPage.body)) {
|
|
result.body.push(...nextPage.body);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return result;
|
|
} catch (err) {
|
|
throw handleGotError(err, url, opts);
|
|
}
|
|
}
|
|
|
|
public async requestGraphql<T = unknown>(
|
|
query: string,
|
|
options: GraphqlOptions = {},
|
|
): Promise<GithubGraphqlResponse<T> | null> {
|
|
const path = 'graphql';
|
|
|
|
const { paginate, count = MAX_GRAPHQL_PAGE_SIZE, cursor = null } = options;
|
|
let { variables } = options;
|
|
if (paginate) {
|
|
variables = {
|
|
...variables,
|
|
count,
|
|
cursor,
|
|
};
|
|
}
|
|
const body = variables ? { query, variables } : { query };
|
|
|
|
const opts: GithubHttpOptions = {
|
|
baseUrl: baseUrl.replace('/v3/', '/'), // GHE uses unversioned graphql path
|
|
body,
|
|
headers: { accept: options?.acceptHeader },
|
|
readOnly: options.readOnly,
|
|
};
|
|
if (options.token) {
|
|
opts.token = options.token;
|
|
}
|
|
logger.trace(`Performing Github GraphQL request`);
|
|
|
|
try {
|
|
const res = await this.postJson<GithubGraphqlResponse<T>>(path, opts);
|
|
return res?.body;
|
|
} catch (err) {
|
|
logger.debug({ err, query, options }, 'Unexpected GraphQL Error');
|
|
if (err instanceof ExternalHostError && count && count > 10) {
|
|
logger.info('Reducing pagination count to workaround graphql errors');
|
|
return null;
|
|
}
|
|
throw handleGotError(err, path, opts);
|
|
}
|
|
}
|
|
|
|
async queryRepoField<T = Record<string, unknown>>(
|
|
query: string,
|
|
fieldName: string,
|
|
options: GraphqlOptions = {},
|
|
): Promise<T[]> {
|
|
const result: T[] = [];
|
|
|
|
const { paginate = true } = options;
|
|
|
|
let optimalCount: null | number = null;
|
|
let count = getGraphqlPageSize(
|
|
fieldName,
|
|
options.count ?? MAX_GRAPHQL_PAGE_SIZE,
|
|
);
|
|
let limit = options.limit ?? 1000;
|
|
let cursor: string | null = null;
|
|
|
|
let isIterating = true;
|
|
while (isIterating) {
|
|
const res = await this.requestGraphql<GithubGraphqlRepoData<T>>(query, {
|
|
...options,
|
|
count: Math.min(count, limit),
|
|
cursor,
|
|
paginate,
|
|
});
|
|
const repositoryData = res?.data?.repository;
|
|
if (
|
|
is.nonEmptyObject(repositoryData) &&
|
|
!is.nullOrUndefined(repositoryData[fieldName])
|
|
) {
|
|
optimalCount = count;
|
|
|
|
const {
|
|
nodes = [],
|
|
edges = [],
|
|
pageInfo,
|
|
} = repositoryData[fieldName] as GraphqlPaginatedContent<T>;
|
|
result.push(...nodes);
|
|
result.push(...edges);
|
|
|
|
limit = Math.max(0, limit - nodes.length - edges.length);
|
|
|
|
if (limit === 0) {
|
|
isIterating = false;
|
|
} else if (paginate && pageInfo) {
|
|
const { hasNextPage, endCursor } = pageInfo;
|
|
if (hasNextPage && endCursor) {
|
|
cursor = endCursor;
|
|
} else {
|
|
isIterating = false;
|
|
}
|
|
}
|
|
} else {
|
|
count = Math.floor(count / 2);
|
|
if (count === 0) {
|
|
logger.warn({ query, options, res }, 'Error fetching GraphQL nodes');
|
|
isIterating = false;
|
|
}
|
|
}
|
|
|
|
if (!paginate) {
|
|
isIterating = false;
|
|
}
|
|
}
|
|
|
|
if (optimalCount && optimalCount < MAX_GRAPHQL_PAGE_SIZE) {
|
|
setGraphqlPageSize(fieldName, optimalCount);
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
/**
|
|
* Get the raw text file from a URL.
|
|
* Only use this method to fetch text files.
|
|
*
|
|
* @param url Full API URL, contents path or path inside the repository to the file
|
|
* @param options
|
|
*
|
|
* @example url = 'https://api.github.com/repos/renovatebot/renovate/contents/package.json'
|
|
* @example url = 'renovatebot/renovate/contents/package.json'
|
|
* @example url = 'package.json' & options.repository = 'renovatebot/renovate'
|
|
*/
|
|
public async getRawTextFile(
|
|
url: string,
|
|
options: InternalHttpOptions & GithubHttpOptions = {},
|
|
): Promise<HttpResponse> {
|
|
const newOptions: InternalHttpOptions & GithubHttpOptions = {
|
|
...options,
|
|
headers: {
|
|
accept: 'application/vnd.github.raw+json',
|
|
},
|
|
};
|
|
|
|
let newURL = url;
|
|
const httpRegex = regEx(/^https?:\/\//);
|
|
if (options.repository && !httpRegex.test(options.repository)) {
|
|
newURL = joinUrlParts(options.repository, 'contents', url);
|
|
}
|
|
|
|
const result = await this.get(newURL, newOptions);
|
|
if (!is.string(result.body)) {
|
|
throw new Error(
|
|
`Expected raw text file but received ${typeof result.body}`,
|
|
);
|
|
}
|
|
return result;
|
|
}
|
|
}
|