mirror of
https://github.com/renovatebot/renovate.git
synced 2025-03-13 07:43:27 +00:00
refactor(datasource/repology): Convert to class (#14132)
Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
This commit is contained in:
Sergei Zharinov
GitHub
parent
e3e286bd01
commit
00e2b51071
3 changed files with 177 additions and 186 deletions
lib/datasource
|
|
@ -31,7 +31,7 @@ import { OrbDatasource } from './orb';
|
|||
import { PackagistDatasource } from './packagist';
|
||||
import { PodDatasource } from './pod';
|
||||
import { PypiDatasource } from './pypi';
|
||||
import * as repology from './repology';
|
||||
import { RepologyDatasource } from './repology';
|
||||
import { RubyVersionDatasource } from './ruby-version';
|
||||
import { RubyGemsDatasource } from './rubygems';
|
||||
import * as sbtPackage from './sbt-package';
|
||||
|
|
@ -76,7 +76,7 @@ api.set(OrbDatasource.id, new OrbDatasource());
|
|||
api.set(PackagistDatasource.id, new PackagistDatasource());
|
||||
api.set(PodDatasource.id, new PodDatasource());
|
||||
api.set(PypiDatasource.id, new PypiDatasource());
|
||||
api.set('repology', repology);
|
||||
api.set(RepologyDatasource.id, new RepologyDatasource());
|
||||
api.set(RubyVersionDatasource.id, new RubyVersionDatasource());
|
||||
api.set(RubyGemsDatasource.id, new RubyGemsDatasource());
|
||||
api.set('sbt-package', sbtPackage);
|
||||
|
|
|
|||
|
|
@ -4,7 +4,9 @@ import { loadFixture } from '../../../test/util';
|
|||
import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages';
|
||||
import { id as versioning } from '../../versioning/loose';
|
||||
import type { RepologyPackage } from './types';
|
||||
import { id as datasource } from '.';
|
||||
import { RepologyDatasource } from './index';
|
||||
|
||||
const datasource = RepologyDatasource.id;
|
||||
|
||||
const repologyHost = 'https://repology.org/';
|
||||
|
||||
|
|
|
|||
|
|
@ -1,76 +1,15 @@
|
|||
import is from '@sindresorhus/is';
|
||||
import { HOST_DISABLED } from '../../constants/error-messages';
|
||||
import { logger } from '../../logger';
|
||||
import { ExternalHostError } from '../../types/errors/external-host-error';
|
||||
import * as packageCache from '../../util/cache/package';
|
||||
import { Http } from '../../util/http';
|
||||
import { cache } from '../../util/cache/package/decorator';
|
||||
import { getQueryString, joinUrlParts } from '../../util/url';
|
||||
import { Datasource } from '../datasource';
|
||||
import type { GetReleasesConfig, ReleaseResult } from '../types';
|
||||
import type { RepologyPackage, RepologyPackageType } from './types';
|
||||
|
||||
export const id = 'repology';
|
||||
export const customRegistrySupport = true;
|
||||
export const defaultRegistryUrls = ['https://repology.org/'];
|
||||
export const registryStrategy = 'hunt';
|
||||
|
||||
const http = new Http(id);
|
||||
const cacheNamespace = `datasource-${id}-list`;
|
||||
const cacheMinutes = 60;
|
||||
|
||||
const packageTypes: RepologyPackageType[] = ['binname', 'srcname'];
|
||||
|
||||
async function queryPackages(url: string): Promise<RepologyPackage[]> {
|
||||
try {
|
||||
const res = await http.getJson<RepologyPackage[]>(url);
|
||||
return res.body;
|
||||
} catch (err) {
|
||||
if (err.statusCode === 404) {
|
||||
// Return an array here because the api does not return proper http codes
|
||||
// and instead of an 404 error an empty array with code 200 is returned
|
||||
// When querying the resolver 404 is thrown if package could not be resolved
|
||||
// and 403 if the repo is not supported
|
||||
// 403 is handled later because in this case we are trying the API
|
||||
return [];
|
||||
}
|
||||
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function queryPackagesViaResolver(
|
||||
registryUrl: string,
|
||||
repoName: string,
|
||||
packageName: string,
|
||||
packageType: RepologyPackageType
|
||||
): Promise<RepologyPackage[]> {
|
||||
const query = getQueryString({
|
||||
repo: repoName,
|
||||
name_type: packageType,
|
||||
target_page: 'api_v1_project',
|
||||
noautoresolve: 'on',
|
||||
name: packageName,
|
||||
});
|
||||
|
||||
// Retrieve list of packages by looking up Repology project
|
||||
const packages = await queryPackages(
|
||||
joinUrlParts(registryUrl, `tools/project-by?${query}`)
|
||||
);
|
||||
|
||||
return packages;
|
||||
}
|
||||
|
||||
async function queryPackagesViaAPI(
|
||||
registryUrl: string,
|
||||
packageName: string
|
||||
): Promise<RepologyPackage[]> {
|
||||
// Directly query the package via the API. This will only work if `packageName` has the
|
||||
// same name as the repology project
|
||||
const packages = await queryPackages(
|
||||
joinUrlParts(registryUrl, `api/v1/project`, packageName)
|
||||
);
|
||||
|
||||
return packages;
|
||||
}
|
||||
|
||||
function findPackageInResponse(
|
||||
response: RepologyPackage[],
|
||||
repoName: string,
|
||||
|
|
@ -104,138 +43,188 @@ function findPackageInResponse(
|
|||
return packagesWithType.length > 0 ? packagesWithType : null;
|
||||
}
|
||||
|
||||
async function queryPackage(
|
||||
registryUrl: string,
|
||||
repoName: string,
|
||||
pkgName: string
|
||||
): Promise<RepologyPackage[]> {
|
||||
let response: RepologyPackage[];
|
||||
let pkg: RepologyPackage[];
|
||||
// Try getting the packages from tools/project-by first for type binname and
|
||||
// afterwards for srcname. This needs to be done first, because some packages
|
||||
// resolve to repology projects which have a different name than the package
|
||||
// e.g. `pulseaudio-utils` resolves to project `pulseaudio`, BUT there is also
|
||||
// a project called `pulseaudio-utils` but it does not contain the package we
|
||||
// are looking for.
|
||||
try {
|
||||
for (const pkgType of packageTypes) {
|
||||
response = await queryPackagesViaResolver(
|
||||
registryUrl,
|
||||
repoName,
|
||||
pkgName,
|
||||
pkgType
|
||||
);
|
||||
export class RepologyDatasource extends Datasource {
|
||||
static readonly id = 'repology';
|
||||
|
||||
if (response) {
|
||||
pkg = findPackageInResponse(response, repoName, pkgName, [pkgType]);
|
||||
if (pkg) {
|
||||
override readonly defaultRegistryUrls = ['https://repology.org/'];
|
||||
|
||||
override readonly registryStrategy = 'hunt';
|
||||
|
||||
constructor() {
|
||||
super(RepologyDatasource.id);
|
||||
}
|
||||
|
||||
private async queryPackages(url: string): Promise<RepologyPackage[]> {
|
||||
try {
|
||||
const res = await this.http.getJson<RepologyPackage[]>(url);
|
||||
return res.body;
|
||||
} catch (err) {
|
||||
if (err.statusCode === 404) {
|
||||
// Return an array here because the api does not return proper http codes
|
||||
// and instead of an 404 error an empty array with code 200 is returned
|
||||
// When querying the resolver 404 is thrown if package could not be resolved
|
||||
// and 403 if the repo is not supported
|
||||
// 403 is handled later because in this case we are trying the API
|
||||
return [];
|
||||
}
|
||||
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
private async queryPackagesViaResolver(
|
||||
registryUrl: string,
|
||||
repoName: string,
|
||||
packageName: string,
|
||||
packageType: RepologyPackageType
|
||||
): Promise<RepologyPackage[]> {
|
||||
const query = getQueryString({
|
||||
repo: repoName,
|
||||
name_type: packageType,
|
||||
target_page: 'api_v1_project',
|
||||
noautoresolve: 'on',
|
||||
name: packageName,
|
||||
});
|
||||
|
||||
// Retrieve list of packages by looking up Repology project
|
||||
const packages = await this.queryPackages(
|
||||
joinUrlParts(registryUrl, `tools/project-by?${query}`)
|
||||
);
|
||||
|
||||
return packages;
|
||||
}
|
||||
|
||||
private async queryPackagesViaAPI(
|
||||
registryUrl: string,
|
||||
packageName: string
|
||||
): Promise<RepologyPackage[]> {
|
||||
// Directly query the package via the API. This will only work if `packageName` has the
|
||||
// same name as the repology project
|
||||
const packages = await this.queryPackages(
|
||||
joinUrlParts(registryUrl, `api/v1/project`, packageName)
|
||||
);
|
||||
|
||||
return packages;
|
||||
}
|
||||
|
||||
@cache({
|
||||
ttlMinutes: 60,
|
||||
namespace: `datasource-${RepologyDatasource.id}-list`,
|
||||
key: (registryUrl: string, repoName: string, pkgName: string) =>
|
||||
joinUrlParts(registryUrl, repoName, pkgName),
|
||||
})
|
||||
async queryPackage(
|
||||
registryUrl: string,
|
||||
repoName: string,
|
||||
pkgName: string
|
||||
): Promise<RepologyPackage[] | undefined> {
|
||||
let response: RepologyPackage[];
|
||||
// Try getting the packages from tools/project-by first for type binname and
|
||||
// afterwards for srcname. This needs to be done first, because some packages
|
||||
// resolve to repology projects which have a different name than the package
|
||||
// e.g. `pulseaudio-utils` resolves to project `pulseaudio`, BUT there is also
|
||||
// a project called `pulseaudio-utils` but it does not contain the package we
|
||||
// are looking for.
|
||||
try {
|
||||
for (const pkgType of packageTypes) {
|
||||
response = await this.queryPackagesViaResolver(
|
||||
registryUrl,
|
||||
repoName,
|
||||
pkgName,
|
||||
pkgType
|
||||
);
|
||||
|
||||
if (response) {
|
||||
const pkg = findPackageInResponse(response, repoName, pkgName, [
|
||||
pkgType,
|
||||
]);
|
||||
if (is.nonEmptyArray(pkg)) {
|
||||
// exit immediately if package found
|
||||
return pkg;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if (err.statusCode === 403) {
|
||||
logger.debug(
|
||||
{ repoName, pkgName },
|
||||
'Repology does not support tools/project-by lookups for repository. Will try direct API access now'
|
||||
);
|
||||
|
||||
// If the repository is not supported in tools/project-by we try directly accessing the
|
||||
// API. This will support all repositories but requires that the project name is equal to the
|
||||
// package name. This won't be always the case but for a good portion we might be able to resolve
|
||||
// the package this way.
|
||||
response = await this.queryPackagesViaAPI(registryUrl, pkgName);
|
||||
const pkg = findPackageInResponse(
|
||||
response,
|
||||
repoName,
|
||||
pkgName,
|
||||
packageTypes
|
||||
);
|
||||
if (is.nonEmptyArray(pkg)) {
|
||||
// exit immediately if package found
|
||||
return pkg;
|
||||
}
|
||||
} else if (err.statusCode === 300) {
|
||||
logger.warn(
|
||||
{ repoName, pkgName },
|
||||
'Ambiguous redirection from package name to project name in Repology. Skipping this package'
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if (err.statusCode === 403) {
|
||||
logger.debug(
|
||||
{ repoName, pkgName },
|
||||
'Repology does not support tools/project-by lookups for repository. Will try direct API access now'
|
||||
);
|
||||
|
||||
// If the repository is not supported in tools/project-by we try directly accessing the
|
||||
// API. This will support all repositories but requires that the project name is equal to the
|
||||
// package name. This won't be always the case but for a good portion we might be able to resolve
|
||||
// the package this way.
|
||||
response = await queryPackagesViaAPI(registryUrl, pkgName);
|
||||
pkg = findPackageInResponse(response, repoName, pkgName, packageTypes);
|
||||
if (pkg) {
|
||||
// exit immediately if package found
|
||||
return pkg;
|
||||
}
|
||||
} else if (err.statusCode === 300) {
|
||||
logger.warn(
|
||||
{ repoName, pkgName },
|
||||
'Ambiguous redirection from package name to project name in Repology. Skipping this package'
|
||||
);
|
||||
return null;
|
||||
throw err;
|
||||
}
|
||||
|
||||
throw err;
|
||||
}
|
||||
|
||||
logger.debug(
|
||||
{ repoName, pkgName },
|
||||
'Repository or package not found on Repology'
|
||||
);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function getCachedPackage(
|
||||
registryUrl: string,
|
||||
repoName: string,
|
||||
pkgName: string
|
||||
): Promise<RepologyPackage[]> {
|
||||
// Fetch previous result from cache if available
|
||||
const cacheKey = joinUrlParts(registryUrl, repoName, pkgName);
|
||||
const cachedResult = await packageCache.get<RepologyPackage[]>(
|
||||
cacheNamespace,
|
||||
cacheKey
|
||||
);
|
||||
// istanbul ignore if
|
||||
if (cachedResult) {
|
||||
return cachedResult;
|
||||
}
|
||||
|
||||
// Attempt a package lookup and return if found non empty list
|
||||
const pkg = await queryPackage(registryUrl, repoName, pkgName);
|
||||
if (pkg && pkg.length > 0) {
|
||||
await packageCache.set(cacheNamespace, cacheKey, pkg, cacheMinutes);
|
||||
return pkg;
|
||||
}
|
||||
|
||||
// No package was found on Repology
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function getReleases({
|
||||
lookupName,
|
||||
registryUrl,
|
||||
}: GetReleasesConfig): Promise<ReleaseResult | null> {
|
||||
// Ensure lookup name contains both repository and package
|
||||
const [repoName, pkgName] = lookupName.split('/', 2);
|
||||
if (!repoName || !pkgName) {
|
||||
throw new ExternalHostError(
|
||||
new Error(
|
||||
'Repology lookup name must contain repository and package separated by slash (<repo>/<pkg>)'
|
||||
)
|
||||
logger.debug(
|
||||
{ repoName, pkgName },
|
||||
'Repository or package not found on Repology'
|
||||
);
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
logger.trace(`repology.getReleases(${repoName}, ${pkgName})`);
|
||||
try {
|
||||
// Attempt to retrieve (cached) package information from Repology
|
||||
const pkg = await getCachedPackage(registryUrl, repoName, pkgName);
|
||||
if (!pkg) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Always prefer origversion if available, otherwise default to version
|
||||
// This is required as source packages usually have no origversion
|
||||
const releases = pkg.map((item) => ({
|
||||
version: item.origversion ?? item.version,
|
||||
}));
|
||||
return { releases };
|
||||
} catch (err) {
|
||||
if (err.message === HOST_DISABLED) {
|
||||
// istanbul ignore next
|
||||
logger.trace({ lookupName, err }, 'Host disabled');
|
||||
} else {
|
||||
logger.warn(
|
||||
{ lookupName, err },
|
||||
'Repology lookup failed with unexpected error'
|
||||
async getReleases({
|
||||
lookupName,
|
||||
registryUrl,
|
||||
}: GetReleasesConfig): Promise<ReleaseResult | null> {
|
||||
// Ensure lookup name contains both repository and package
|
||||
const [repoName, pkgName] = lookupName.split('/', 2);
|
||||
if (!repoName || !pkgName) {
|
||||
throw new ExternalHostError(
|
||||
new Error(
|
||||
'Repology lookup name must contain repository and package separated by slash (<repo>/<pkg>)'
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
throw new ExternalHostError(err);
|
||||
logger.trace(`repology.getReleases(${repoName}, ${pkgName})`);
|
||||
try {
|
||||
// Attempt to retrieve (cached) package information from Repology
|
||||
const pkg = await this.queryPackage(registryUrl, repoName, pkgName);
|
||||
if (!pkg) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Always prefer origversion if available, otherwise default to version
|
||||
// This is required as source packages usually have no origversion
|
||||
const releases = pkg.map((item) => ({
|
||||
version: item.origversion ?? item.version,
|
||||
}));
|
||||
return { releases };
|
||||
} catch (err) {
|
||||
if (err.message === HOST_DISABLED) {
|
||||
// istanbul ignore next
|
||||
logger.trace({ lookupName, err }, 'Host disabled');
|
||||
} else {
|
||||
logger.warn(
|
||||
{ lookupName, err },
|
||||
'Repology lookup failed with unexpected error'
|
||||
);
|
||||
}
|
||||
|
||||
throw new ExternalHostError(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue