feat(github): long-term datasource caching (#15653)

lib/modules/datasource/github-releases/__snapshots__/index.spec.ts.snap

@@ -1,64 +0,0 @@
-// Jest Snapshot v1, https://goo.gl/fbAQLP
-
-exports[`modules/datasource/github-releases/index getReleases returns releases 1`] = `
-Object {
-  "registryUrl": "https://github.com",
-  "releases": Array [
-    Object {
-      "gitRef": "1.0.0",
-      "releaseTimestamp": "2020-03-09T11:00:00.000Z",
-      "version": "1.0.0",
-    },
-    Object {
-      "gitRef": "v1.1.0",
-      "releaseTimestamp": "2020-03-09T10:00:00.000Z",
-      "version": "v1.1.0",
-    },
-    Object {
-      "gitRef": "2.0.0",
-      "isStable": false,
-      "releaseTimestamp": "2020-04-09T10:00:00.000Z",
-      "version": "2.0.0",
-    },
-  ],
-  "sourceUrl": "https://github.com/some/dep",
-}
-`;
-
-exports[`modules/datasource/github-releases/index getReleases supports ghe 1`] = `
-Object {
-  "releases": Array [
-    Object {
-      "gitRef": "a",
-      "isStable": undefined,
-      "releaseTimestamp": "2020-03-09T13:00:00Z",
-      "version": "a",
-    },
-    Object {
-      "gitRef": "v",
-      "isStable": undefined,
-      "releaseTimestamp": "2020-03-09T12:00:00Z",
-      "version": "v",
-    },
-    Object {
-      "gitRef": "1.0.0",
-      "isStable": undefined,
-      "releaseTimestamp": "2020-03-09T11:00:00Z",
-      "version": "1.0.0",
-    },
-    Object {
-      "gitRef": "v1.1.0",
-      "isStable": undefined,
-      "releaseTimestamp": "2020-03-09T10:00:00Z",
-      "version": "v1.1.0",
-    },
-    Object {
-      "gitRef": "2.0.0",
-      "isStable": false,
-      "releaseTimestamp": "2020-04-09T10:00:00Z",
-      "version": "2.0.0",
-    },
-  ],
-  "sourceUrl": "https://git.enterprise.com/some/dep",
-}
-`;

lib/modules/datasource/github-releases/cache/cache-base.spec.ts

@@ -0,0 +1,262 @@
+import { DateTime } from 'luxon';
+import { mocked } from '../../../../../test/util';
+import * as _packageCache from '../../../../util/cache/package';
+import {
+  GithubGraphqlResponse,
+  GithubHttp,
+} from '../../../../util/http/github';
+import { AbstractGithubDatasourceCache } from './cache-base';
+import type { QueryResponse, StoredItemBase } from './types';
+
+jest.mock('../../../../util/cache/package');
+const packageCache = mocked(_packageCache);
+
+interface FetchedItem {
+  name: string;
+  createdAt: string;
+  foo: string;
+}
+
+interface StoredItem extends StoredItemBase {
+  bar: string;
+}
+
+type GraphqlDataResponse = {
+  statusCode: 200;
+  headers: Record<string, string>;
+  body: GithubGraphqlResponse<QueryResponse<FetchedItem>>;
+};
+
+type GraphqlResponse = GraphqlDataResponse | Error;
+
+class TestCache extends AbstractGithubDatasourceCache<StoredItem, FetchedItem> {
+  cacheNs = 'test-cache';
+  graphqlQuery = `query { ... }`;
+
+  coerceFetched({
+    name: version,
+    createdAt: releaseTimestamp,
+    foo: bar,
+  }: FetchedItem): StoredItem | null {
+    return version === 'invalid' ? null : { version, releaseTimestamp, bar };
+  }
+
+  isEquivalent({ bar: x }: StoredItem, { bar: y }: StoredItem): boolean {
+    return x === y;
+  }
+}
+
+function resp(items: FetchedItem[], hasNextPage = false): GraphqlDataResponse {
+  return {
+    statusCode: 200,
+    headers: {},
+    body: {
+      data: {
+        repository: {
+          payload: {
+            nodes: items,
+            pageInfo: {
+              hasNextPage,
+              endCursor: 'abc',
+            },
+          },
+        },
+      },
+    },
+  };
+}
+
+const sortItems = (items: StoredItem[]) =>
+  items.sort(({ releaseTimestamp: x }, { releaseTimestamp: y }) =>
+    x.localeCompare(y)
+  );
+
+describe('modules/datasource/github-releases/cache/cache-base', () => {
+  const http = new GithubHttp();
+  const httpPostJson = jest.spyOn(GithubHttp.prototype, 'postJson');
+
+  const now = DateTime.local(2022, 6, 15, 18, 30, 30);
+  const t1 = now.minus({ days: 3 }).toISO();
+  const t2 = now.minus({ days: 2 }).toISO();
+  const t3 = now.minus({ days: 1 }).toISO();
+
+  let responses: GraphqlResponse[] = [];
+
+  beforeEach(() => {
+    responses = [];
+    jest.resetAllMocks();
+    jest.spyOn(DateTime, 'now').mockReturnValue(now);
+    httpPostJson.mockImplementation(() => {
+      const resp = responses.shift();
+      return resp instanceof Error
+        ? Promise.reject(resp)
+        : Promise.resolve(resp);
+    });
+  });
+
+  it('performs pre-fetch', async () => {
+    responses = [
+      resp([{ name: 'v3', createdAt: t3, foo: 'ccc' }], true),
+      resp([{ name: 'v2', createdAt: t2, foo: 'bbb' }], true),
+      resp([{ name: 'v1', createdAt: t1, foo: 'aaa' }]),
+    ];
+    const cache = new TestCache(http, { resetDeltaMinutes: 0 });
+
+    const res = await cache.getItems({ packageName: 'foo/bar' });
+
+    expect(sortItems(res)).toMatchObject([
+      { version: 'v1', bar: 'aaa' },
+      { version: 'v2', bar: 'bbb' },
+      { version: 'v3', bar: 'ccc' },
+    ]);
+    expect(packageCache.set).toHaveBeenCalledWith(
+      'test-cache',
+      'https://api.github.com/:foo:bar',
+      {
+        createdAt: now.toISO(),
+        updatedAt: now.toISO(),
+        items: {
+          v1: { bar: 'aaa', releaseTimestamp: t1, version: 'v1' },
+          v2: { bar: 'bbb', releaseTimestamp: t2, version: 'v2' },
+          v3: { bar: 'ccc', releaseTimestamp: t3, version: 'v3' },
+        },
+      },
+      7 * 24 * 60
+    );
+  });
+
+  it('filters out items being coerced to null', async () => {
+    responses = [
+      resp([{ name: 'v3', createdAt: t3, foo: 'ccc' }], true),
+      resp([{ name: 'invalid', createdAt: t3, foo: 'xxx' }], true),
+      resp([{ name: 'v2', createdAt: t2, foo: 'bbb' }], true),
+      resp([{ name: 'v1', createdAt: t1, foo: 'aaa' }]),
+    ];
+    const cache = new TestCache(http, { resetDeltaMinutes: 0 });
+
+    const res = await cache.getItems({ packageName: 'foo/bar' });
+
+    expect(sortItems(res)).toMatchObject([
+      { version: 'v1' },
+      { version: 'v2' },
+      { version: 'v3' },
+    ]);
+  });
+
+  it('updates items', async () => {
+    packageCache.get.mockResolvedValueOnce({
+      items: {
+        v1: { version: 'v1', releaseTimestamp: t1, bar: 'aaa' },
+        v2: { version: 'v2', releaseTimestamp: t2, bar: 'bbb' },
+        v3: { version: 'v3', releaseTimestamp: t3, bar: 'ccc' },
+      },
+      createdAt: t3,
+      updatedAt: t3,
+    });
+
+    responses = [
+      resp([{ name: 'v3', createdAt: t3, foo: 'xxx' }], true),
+      resp([{ name: 'v2', createdAt: t2, foo: 'bbb' }], true),
+      resp([{ name: 'v1', createdAt: t1, foo: 'aaa' }]),
+    ];
+    const cache = new TestCache(http, { resetDeltaMinutes: 0 });
+
+    const res = await cache.getItems({ packageName: 'foo/bar' });
+
+    expect(sortItems(res)).toMatchObject([
+      { version: 'v1', bar: 'aaa' },
+      { version: 'v2', bar: 'bbb' },
+      { version: 'v3', bar: 'xxx' },
+    ]);
+    expect(packageCache.set).toHaveBeenCalledWith(
+      'test-cache',
+      'https://api.github.com/:foo:bar',
+      {
+        createdAt: t3,
+        updatedAt: now.toISO(),
+        items: {
+          v1: { bar: 'aaa', releaseTimestamp: t1, version: 'v1' },
+          v2: { bar: 'bbb', releaseTimestamp: t2, version: 'v2' },
+          v3: { bar: 'xxx', releaseTimestamp: t3, version: 'v3' },
+        },
+      },
+      6 * 24 * 60
+    );
+  });
+
+  it('stops updating once stability period have passed', async () => {
+    packageCache.get.mockResolvedValueOnce({
+      items: {
+        v1: { version: 'v1', releaseTimestamp: t1, bar: 'aaa' },
+        v2: { version: 'v2', releaseTimestamp: t2, bar: 'bbb' },
+        v3: { version: 'v3', releaseTimestamp: t3, bar: 'ccc' },
+      },
+      createdAt: t3,
+      updatedAt: t3,
+    });
+    responses = [
+      resp([{ name: 'v3', createdAt: t3, foo: 'zzz' }], true),
+      resp([{ name: 'v2', createdAt: t2, foo: 'yyy' }], true),
+      resp([{ name: 'v1', createdAt: t1, foo: 'xxx' }]),
+    ];
+    const cache = new TestCache(http, { unstableDays: 1.5 });
+
+    const res = await cache.getItems({ packageName: 'foo/bar' });
+
+    expect(sortItems(res)).toMatchObject([
+      { version: 'v1', bar: 'aaa' },
+      { version: 'v2', bar: 'bbb' },
+      { version: 'v3', bar: 'zzz' },
+    ]);
+  });
+
+  it('removes deleted items from cache', async () => {
+    packageCache.get.mockResolvedValueOnce({
+      items: {
+        v1: { version: 'v1', releaseTimestamp: t1, bar: 'aaa' },
+        v2: { version: 'v2', releaseTimestamp: t2, bar: 'bbb' },
+        v3: { version: 'v3', releaseTimestamp: t3, bar: 'ccc' },
+      },
+      createdAt: t3,
+      updatedAt: t3,
+    });
+    responses = [
+      resp([{ name: 'v3', createdAt: t3, foo: 'ccc' }], true),
+      resp([{ name: 'v1', createdAt: t1, foo: 'aaa' }]),
+    ];
+    const cache = new TestCache(http, { resetDeltaMinutes: 0 });
+
+    const res = await cache.getItems({ packageName: 'foo/bar' });
+
+    expect(sortItems(res)).toMatchObject([
+      { version: 'v1', bar: 'aaa' },
+      { version: 'v3', bar: 'ccc' },
+    ]);
+  });
+
+  it('returns cached values on server errors', async () => {
+    packageCache.get.mockResolvedValueOnce({
+      items: {
+        v1: { version: 'v1', releaseTimestamp: t1, bar: 'aaa' },
+        v2: { version: 'v2', releaseTimestamp: t2, bar: 'bbb' },
+        v3: { version: 'v3', releaseTimestamp: t3, bar: 'ccc' },
+      },
+      createdAt: t3,
+      updatedAt: t3,
+    });
+    responses = [
+      resp([{ name: 'v3', createdAt: t3, foo: 'zzz' }], true),
+      new Error('Unknown error'),
+      resp([{ name: 'v1', createdAt: t1, foo: 'xxx' }]),
+    ];
+    const cache = new TestCache(http, { resetDeltaMinutes: 0 });
+
+    const res = await cache.getItems({ packageName: 'foo/bar' });
+
+    expect(sortItems(res)).toMatchObject([
+      { version: 'v1', bar: 'aaa' },
+      { version: 'v2', bar: 'bbb' },
+      { version: 'v3', bar: 'ccc' },
+    ]);
+  });
+});

lib/modules/datasource/github-releases/cache/cache-base.ts

@@ -0,0 +1,301 @@
+import { DateTime, DurationLikeObject } from 'luxon';
+import { logger } from '../../../../logger';
+import * as packageCache from '../../../../util/cache/package';
+import type {
+  GithubGraphqlResponse,
+  GithubHttp,
+} from '../../../../util/http/github';
+import type { GetReleasesConfig } from '../../types';
+import { getApiBaseUrl } from '../common';
+import type {
+  CacheOptions,
+  GithubDatasourceCache,
+  GithubQueryParams,
+  QueryResponse,
+  StoredItemBase,
+} from './types';
+
+/**
+ * The options that are meant to be used in production.
+ */
+const cacheDefaults: Required<CacheOptions> = {
+  /**
+   * How many minutes to wait until next cache update
+   */
+  updateAfterMinutes: 30,
+
+  /**
+   * How many days to wait until full cache reset (for single package).
+   */
+  resetAfterDays: 7,
+
+  /**
+   * Delays cache reset by some random amount of minutes,
+   * in order to stabilize load during mass cache reset.
+   */
+  resetDeltaMinutes: 3 * 60,
+
+  /**
+   * How many days ago the package should be published to be considered as stable.
+   * Since this period is expired, it won't be refreshed via soft updates anymore.
+   */
+  unstableDays: 30,
+
+  /**
+   * How many items per page to obtain per page during initial fetch (i.e. pre-fetch)
+   */
+  itemsPerPrefetchPage: 100,
+
+  /**
+   * How many pages to fetch (at most) during the initial fetch (i.e. pre-fetch)
+   */
+  maxPrefetchPages: 100,
+
+  /**
+   * How many items per page to obtain per page during the soft update
+   */
+  itemsPerUpdatePage: 100,
+
+  /**
+   * How many pages to fetch (at most) during the soft update
+   */
+  maxUpdatePages: 100,
+};
+
+/**
+ * Tells whether the time `duration` is expired starting
+ * from the `date` (ISO date format) at the moment of `now`.
+ */
+function isExpired(
+  now: DateTime,
+  date: string,
+  duration: DurationLikeObject
+): boolean {
+  const then = DateTime.fromISO(date);
+  const expiry = then.plus(duration);
+  return now >= expiry;
+}
+
+export abstract class AbstractGithubDatasourceCache<
+  StoredItem extends StoredItemBase,
+  FetchedItem = unknown
+> {
+  private updateDuration: DurationLikeObject;
+  private resetDuration: DurationLikeObject;
+  private stabilityDuration: DurationLikeObject;
+
+  private maxPrefetchPages: number;
+  private itemsPerPrefetchPage: number;
+
+  private maxUpdatePages: number;
+  private itemsPerUpdatePage: number;
+
+  private resetDeltaMinutes: number;
+
+  constructor(private http: GithubHttp, opts: CacheOptions = {}) {
+    const {
+      updateAfterMinutes,
+      resetAfterDays,
+      unstableDays,
+      maxPrefetchPages,
+      itemsPerPrefetchPage,
+      maxUpdatePages,
+      itemsPerUpdatePage,
+      resetDeltaMinutes,
+    } = {
+      ...cacheDefaults,
+      ...opts,
+    };
+
+    this.updateDuration = { minutes: updateAfterMinutes };
+    this.resetDuration = { days: resetAfterDays };
+    this.stabilityDuration = { days: unstableDays };
+
+    this.maxPrefetchPages = maxPrefetchPages;
+    this.itemsPerPrefetchPage = itemsPerPrefetchPage;
+    this.maxUpdatePages = maxUpdatePages;
+    this.itemsPerUpdatePage = itemsPerUpdatePage;
+
+    this.resetDeltaMinutes = resetDeltaMinutes;
+  }
+
+  /**
+   * The key at which data is stored in the package cache.
+   */
+  abstract readonly cacheNs: string;
+
+  /**
+   * The query string.
+   * For parameters, see `GithubQueryParams`.
+   */
+  abstract readonly graphqlQuery: string;
+
+  /**
+   * Transform `fetchedItem` for storing in the package cache.
+   * @param fetchedItem Node obtained from GraphQL response
+   */
+  abstract coerceFetched(fetchedItem: FetchedItem): StoredItem | null;
+
+  /**
+   * Pre-fetch, update, or just return the package cache items.
+   */
+  async getItems(releasesConfig: GetReleasesConfig): Promise<StoredItem[]> {
+    const { packageName, registryUrl } = releasesConfig;
+
+    // The time meant to be used across the function
+    const now = DateTime.now();
+
+    // Initialize items and timestamps for the new cache
+    let cacheItems: Record<string, StoredItem> = {};
+
+    // Add random minutes to the creation date in order to
+    // provide back-off time during mass cache invalidation.
+    const randomDelta = this.getRandomDeltaMinutes();
+    let cacheCreatedAt = now.plus(randomDelta).toISO();
+
+    // We have to initialize `updatedAt` value as already expired,
+    // so that soft update mechanics is immediately starting.
+    let cacheUpdatedAt = now.minus(this.updateDuration).toISO();
+
+    const baseUrl = getApiBaseUrl(registryUrl).replace('/v3/', '/'); // Replace for GHE
+
+    const [owner, name] = packageName.split('/');
+    if (owner && name) {
+      const cacheKey = `${baseUrl}:${owner}:${name}`;
+      const cache = await packageCache.get<GithubDatasourceCache<StoredItem>>(
+        this.cacheNs,
+        cacheKey
+      );
+
+      const cacheDoesExist =
+        cache && !isExpired(now, cache.createdAt, this.resetDuration);
+      if (cacheDoesExist) {
+        // Keeping the the original `cache` value intact
+        // in order to be used in exception handler
+        cacheItems = { ...cache.items };
+        cacheCreatedAt = cache.createdAt;
+        cacheUpdatedAt = cache.updatedAt;
+      }
+
+      try {
+        if (isExpired(now, cacheUpdatedAt, this.updateDuration)) {
+          const variables: GithubQueryParams = {
+            owner,
+            name,
+            cursor: null,
+            count: cacheDoesExist
+              ? this.itemsPerUpdatePage
+              : this.itemsPerPrefetchPage,
+          };
+
+          // Collect version values to determine deleted items
+          const checkedVersions = new Set<string>();
+
+          // Page-by-page update loop
+          let pagesRemained = cacheDoesExist
+            ? this.maxUpdatePages
+            : this.maxPrefetchPages;
+          let stopIteration = false;
+          while (pagesRemained > 0 && !stopIteration) {
+            const graphqlRes = await this.http.postJson<
+              GithubGraphqlResponse<QueryResponse<FetchedItem>>
+            >('/graphql', {
+              baseUrl,
+              body: { query: this.graphqlQuery, variables },
+            });
+            pagesRemained -= 1;
+
+            const data = graphqlRes.body.data;
+            if (data) {
+              const {
+                nodes: fetchedItems,
+                pageInfo: { hasNextPage, endCursor },
+              } = data.repository.payload;
+
+              if (hasNextPage) {
+                variables.cursor = endCursor;
+              } else {
+                stopIteration = true;
+              }
+
+              for (const item of fetchedItems) {
+                const newStoredItem = this.coerceFetched(item);
+                if (newStoredItem) {
+                  const { version } = newStoredItem;
+
+                  // Stop earlier if the stored item have reached stability,
+                  // which means `unstableDays` period have passed
+                  const oldStoredItem = cacheItems[version];
+                  if (
+                    oldStoredItem &&
+                    isExpired(
+                      now,
+                      oldStoredItem.releaseTimestamp,
+                      this.stabilityDuration
+                    )
+                  ) {
+                    stopIteration = true;
+                    break;
+                  }
+
+                  cacheItems[version] = newStoredItem;
+                  checkedVersions.add(version);
+                }
+              }
+            }
+          }
+
+          // Detect removed items
+          for (const [version, item] of Object.entries(cacheItems)) {
+            if (
+              !isExpired(now, item.releaseTimestamp, this.stabilityDuration) &&
+              !checkedVersions.has(version)
+            ) {
+              delete cacheItems[version];
+            }
+          }
+
+          // Store cache
+          const expiry = DateTime.fromISO(cacheCreatedAt).plus(
+            this.resetDuration
+          );
+          const { minutes: ttlMinutes } = expiry
+            .diff(now, ['minutes'])
+            .toObject();
+          if (ttlMinutes && ttlMinutes > 0) {
+            const cacheValue: GithubDatasourceCache<StoredItem> = {
+              items: cacheItems,
+              createdAt: cacheCreatedAt,
+              updatedAt: now.toISO(),
+            };
+            await packageCache.set(
+              this.cacheNs,
+              cacheKey,
+              cacheValue,
+              ttlMinutes
+            );
+          }
+        }
+      } catch (err) {
+        logger.debug(
+          { err },
+          `GitHub datasource: error fetching cacheable GraphQL data`
+        );
+
+        // On errors, return previous value (if valid)
+        if (cacheDoesExist) {
+          const cachedItems = Object.values(cache.items);
+          return cachedItems;
+        }
+      }
+    }
+
+    const items = Object.values(cacheItems);
+    return items;
+  }
+
+  getRandomDeltaMinutes(): number {
+    const rnd = Math.random();
+    return Math.floor(rnd * this.resetDeltaMinutes);
+  }
+}

lib/modules/datasource/github-releases/cache/index.spec.ts

@@ -0,0 +1,43 @@
+import { GithubHttp } from '../../../../util/http/github';
+import { CacheableGithubReleases, FetchedRelease } from '.';
+
+describe('modules/datasource/github-releases/cache/index', () => {
+  const http = new GithubHttp();
+  const cache = new CacheableGithubReleases(http, { resetDeltaMinutes: 0 });
+
+  const fetchedItem: FetchedRelease = {
+    version: '1.2.3',
+    releaseTimestamp: '2020-04-09T10:00:00.000Z',
+    isDraft: false,
+    isPrerelease: false,
+    url: 'https://example.com/',
+    id: 123,
+    name: 'Some name',
+    description: 'Some description',
+  };
+
+  describe('coerceFetched', () => {
+    it('transforms GraphQL item', () => {
+      expect(cache.coerceFetched(fetchedItem)).toEqual({
+        description: 'Some description',
+        id: 123,
+        name: 'Some name',
+        releaseTimestamp: '2020-04-09T10:00:00.000Z',
+        url: 'https://example.com/',
+        version: '1.2.3',
+      });
+    });
+
+    it('marks pre-release as unstable', () => {
+      expect(
+        cache.coerceFetched({ ...fetchedItem, isPrerelease: true })
+      ).toMatchObject({
+        isStable: false,
+      });
+    });
+
+    it('filters out drafts', () => {
+      expect(cache.coerceFetched({ ...fetchedItem, isDraft: true })).toBeNull();
+    });
+  });
+});

lib/modules/datasource/github-releases/cache/index.ts

@@ -0,0 +1,93 @@
+import type { GithubHttp } from '../../../../util/http/github';
+import { AbstractGithubDatasourceCache } from './cache-base';
+import type { CacheOptions, StoredItemBase } from './types';
+
+export const query = `
+query ($owner: String!, $name: String!, $cursor: String, $count: Int!) {
+  repository(owner: $owner, name: $name) {
+    payload: releases(
+      first: $count
+      after: $cursor
+      orderBy: {field: CREATED_AT, direction: DESC}
+    ) {
+      nodes {
+        version: tagName
+        releaseTimestamp: publishedAt
+        isDraft
+        isPrerelease
+        url
+        id: databaseId
+        name
+        description
+      }
+      pageInfo {
+        hasNextPage
+        endCursor
+      }
+    }
+  }
+}
+`;
+
+export interface FetchedRelease {
+  version: string;
+  releaseTimestamp: string;
+  isDraft: boolean;
+  isPrerelease: boolean;
+  url: string;
+  id: number;
+  name: string;
+  description: string;
+}
+
+export interface StoredRelease extends StoredItemBase {
+  isStable?: boolean;
+  url: string;
+  id: number;
+  name: string;
+  description: string;
+}
+
+export class CacheableGithubReleases extends AbstractGithubDatasourceCache<
+  StoredRelease,
+  FetchedRelease
+> {
+  cacheNs = 'github-datasource-graphql-releases';
+  graphqlQuery = query;
+
+  constructor(http: GithubHttp, opts: CacheOptions = {}) {
+    super(http, opts);
+  }
+
+  coerceFetched(item: FetchedRelease): StoredRelease | null {
+    const {
+      version,
+      releaseTimestamp,
+      isDraft,
+      isPrerelease,
+      url,
+      id,
+      name,
+      description,
+    } = item;
+
+    if (isDraft) {
+      return null;
+    }
+
+    const result: StoredRelease = {
+      version,
+      releaseTimestamp,
+      url,
+      id,
+      name,
+      description,
+    };
+
+    if (isPrerelease) {
+      result.isStable = false;
+    }
+
+    return result;
+  }
+}

lib/modules/datasource/github-releases/cache/types.ts

@@ -0,0 +1,100 @@
+/**
+ * Every `AbstractGithubDatasourceCache` implementation
+ * should have `graphqlQuery` that uses parameters
+ * defined this interface.
+ */
+export interface GithubQueryParams {
+  owner: string;
+  name: string;
+  cursor: string | null;
+  count: number;
+}
+
+/**
+ * Every `AbstractGithubDatasourceCache` implementation
+ * should have `graphqlQuery` that resembles the structure
+ * of this interface.
+ */
+export interface QueryResponse<T = unknown> {
+  repository: {
+    payload: {
+      nodes: T[];
+      pageInfo: {
+        hasNextPage: boolean;
+        endCursor: string;
+      };
+    };
+  };
+}
+
+/**
+ * Base interface meant to be extended by all implementations.
+ * Must have `version` and `releaseTimestamp` fields.
+ */
+export interface StoredItemBase {
+  /** The values of `version` field meant to be unique. */
+  version: string;
+
+  /** The `releaseTimestamp` field meant to be ISO-encoded date. */
+  releaseTimestamp: string;
+}
+
+/**
+ * The data structure stored in the package cache.
+ */
+export interface GithubDatasourceCache<StoredItem extends StoredItemBase> {
+  items: Record<string, StoredItem>;
+
+  /** Cache full reset decision is based on `createdAt` value. */
+  createdAt: string;
+
+  /** Cache soft updates are performed depending on `updatedAt` value. */
+  updatedAt: string;
+}
+
+/**
+ * The configuration for cache.
+ */
+export interface CacheOptions {
+  /**
+   * How many minutes to wait until next cache update
+   */
+  updateAfterMinutes?: number;
+
+  /**
+   * How many days to wait until full cache reset (for single package).
+   */
+  resetAfterDays?: number;
+
+  /**
+   * Delays cache reset by some random amount of minutes,
+   * in order to stabilize load during mass cache reset.
+   */
+  resetDeltaMinutes?: number;
+
+  /**
+   * How many days ago the package should be published to be considered as stable.
+   * Since this period is expired, it won't be refreshed via soft updates anymore.
+   */
+  unstableDays?: number;
+
+  /**
+   * How many items per page to obtain per page during initial fetch (i.e. pre-fetch)
+   */
+  itemsPerPrefetchPage?: number;
+
+  /**
+   * How many pages to fetch (at most) during the initial fetch (i.e. pre-fetch)
+   */
+  maxPrefetchPages?: number;
+
+  /**
+   * How many items per page to obtain per page during the soft update
+   */
+  itemsPerUpdatePage?: number;
+
+  /**
+   * How many pages to fetch (at most) during the soft update
+   */
+  maxUpdatePages?: number;
+}

lib/modules/datasource/github-releases/common.ts

@@ -1,6 +1,7 @@
 import { ensureTrailingSlash } from '../../../util/url';
 
 const defaultSourceUrlBase = 'https://github.com/';
+const defaultApiBaseUrl = 'https://api.github.com/';
 
 export function getSourceUrlBase(registryUrl: string | undefined): string {
   // default to GitHub.com if no GHE host is specified.
@@ -9,8 +10,8 @@ export function getSourceUrlBase(registryUrl: string | undefined): string {
 
 export function getApiBaseUrl(registryUrl: string | undefined): string {
   const sourceUrlBase = getSourceUrlBase(registryUrl);
-  return sourceUrlBase === defaultSourceUrlBase
+  return [defaultSourceUrlBase, defaultApiBaseUrl].includes(sourceUrlBase)
-    ? `https://api.github.com/`
+    ? defaultApiBaseUrl
     : `${sourceUrlBase}api/v3/`;
 }
 

lib/modules/datasource/github-releases/index.spec.ts

@@ -1,6 +1,6 @@
 import { getDigest, getPkgReleases } from '..';
-import * as httpMock from '../../../../test/http-mock';
 import * as _hostRules from '../../../util/host-rules';
+import { CacheableGithubReleases } from './cache';
 import { GitHubReleaseMocker } from './test';
 import { GithubReleasesDatasource } from '.';
 
@@ -8,25 +8,15 @@ jest.mock('../../../util/host-rules');
 const hostRules: any = _hostRules;
 
 const githubApiHost = 'https://api.github.com';
-const githubEnterpriseApiHost = 'https://git.enterprise.com';
-
-const responseBody = [
-  { tag_name: 'a', published_at: '2020-03-09T13:00:00Z' },
-  { tag_name: 'v', published_at: '2020-03-09T12:00:00Z' },
-  { tag_name: '1.0.0', published_at: '2020-03-09T11:00:00Z' },
-  { tag_name: 'v1.1.0', draft: false, published_at: '2020-03-09T10:00:00Z' },
-  { tag_name: '1.2.0', draft: true, published_at: '2020-03-09T10:00:00Z' },
-  {
-    tag_name: '2.0.0',
-    published_at: '2020-04-09T10:00:00Z',
-    prerelease: true,
-  },
-];
 
 describe('modules/datasource/github-releases/index', () => {
-  const githubReleases = new GithubReleasesDatasource();
+  const cacheGetItems = jest.spyOn(
+    CacheableGithubReleases.prototype,
+    'getItems'
+  );
 
   beforeEach(() => {
+    jest.resetAllMocks();
     hostRules.hosts.mockReturnValue([]);
     hostRules.find.mockReturnValue({
       token: 'some-token',
@@ -35,39 +25,36 @@ describe('modules/datasource/github-releases/index', () => {
 
   describe('getReleases', () => {
     it('returns releases', async () => {
-      httpMock
+      cacheGetItems.mockResolvedValueOnce([
-        .scope(githubApiHost)
+        { version: 'a', releaseTimestamp: '2020-03-09T13:00:00Z' },
-        .get('/repos/some/dep/releases?per_page=100')
+        { version: 'v', releaseTimestamp: '2020-03-09T12:00:00Z' },
-        .reply(200, responseBody);
+        { version: '1.0.0', releaseTimestamp: '2020-03-09T11:00:00Z' },
+        { version: 'v1.1.0', releaseTimestamp: '2020-03-09T10:00:00Z' },
+        {
+          version: '2.0.0',
+          releaseTimestamp: '2020-04-09T10:00:00Z',
+          isStable: false,
+        },
+      ] as never);
 
       const res = await getPkgReleases({
         datasource: GithubReleasesDatasource.id,
         depName: 'some/dep',
       });
-      expect(res).toMatchSnapshot();
-      expect(res.releases).toHaveLength(3);
-      expect(
-        res.releases.find((release) => release.version === 'v1.1.0')
-      ).toBeDefined();
-      expect(
-        res.releases.find((release) => release.version === '1.2.0')
-      ).toBeUndefined();
-      expect(
-        res.releases.find((release) => release.version === '2.0.0').isStable
-      ).toBeFalse();
-    });
 
-    it('supports ghe', async () => {
+      expect(res).toMatchObject({
-      const packageName = 'some/dep';
+        registryUrl: 'https://github.com',
-      httpMock
+        releases: [
-        .scope(githubEnterpriseApiHost)
+          { releaseTimestamp: '2020-03-09T11:00:00.000Z', version: '1.0.0' },
-        .get(`/api/v3/repos/${packageName}/releases?per_page=100`)
+          { version: 'v1.1.0', releaseTimestamp: '2020-03-09T10:00:00.000Z' },
-        .reply(200, responseBody);
+          {
-      const res = await githubReleases.getReleases({
+            version: '2.0.0',
-        registryUrl: 'https://git.enterprise.com',
+            releaseTimestamp: '2020-04-09T10:00:00.000Z',
-        packageName,
+            isStable: false,
+          },
+        ],
+        sourceUrl: 'https://github.com/some/dep',
       });
-      expect(res).toMatchSnapshot();
     });
   });
 

lib/modules/datasource/github-releases/index.ts

@@ -4,7 +4,13 @@ import { cache } from '../../../util/cache/package/decorator';
 import { GithubHttp } from '../../../util/http/github';
 import { newlineRegex, regEx } from '../../../util/regex';
 import { Datasource } from '../datasource';
-import type { DigestConfig, GetReleasesConfig, ReleaseResult } from '../types';
+import type {
+  DigestConfig,
+  GetReleasesConfig,
+  Release,
+  ReleaseResult,
+} from '../types';
+import { CacheableGithubReleases } from './cache';
 import { getApiBaseUrl, getSourceUrl } from './common';
 import type { DigestAsset, GithubRelease, GithubReleaseAsset } from './types';
 
@@ -27,9 +33,12 @@ export class GithubReleasesDatasource extends Datasource {
 
   override http: GithubHttp;
 
+  private releasesCache: CacheableGithubReleases;
+
   constructor(id = GithubReleasesDatasource.id) {
     super(id);
     this.http = new GithubHttp(id);
+    this.releasesCache = new CacheableGithubReleases(this.http);
   }
 
   async findDigestFile(
@@ -218,11 +227,6 @@ export class GithubReleasesDatasource extends Datasource {
     return newDigest;
   }
 
-  @cache({
-    namespace: 'datasource-github-releases',
-    key: ({ packageName: repo, registryUrl }: GetReleasesConfig) =>
-      `${registryUrl}:${repo}:tags`,
-  })
   /**
    * github.getReleases
    *
@@ -233,27 +237,22 @@ export class GithubReleasesDatasource extends Datasource {
    *  - Sanitize the versions if desired (e.g. strip out leading 'v')
    *  - Return a dependency object containing sourceUrl string and releases array
    */
-  async getReleases({
+  async getReleases(config: GetReleasesConfig): Promise<ReleaseResult | null> {
-    packageName: repo,
+    let result: ReleaseResult | null = null;
-    registryUrl,
+    const releases = await this.releasesCache.getItems(config);
-  }: GetReleasesConfig): Promise<ReleaseResult | null> {
+    if (releases.length) {
-    const apiBaseUrl = getApiBaseUrl(registryUrl);
+      result = {
-    const url = `${apiBaseUrl}repos/${repo}/releases?per_page=100`;
+        sourceUrl: getSourceUrl(config.packageName, config.registryUrl),
-    const res = await this.http.getJson<GithubRelease[]>(url, {
+        releases: releases.map((item) => {
-      paginate: true,
+          const { version, releaseTimestamp, isStable } = item;
-    });
+          const result: Release = { version, releaseTimestamp };
-    const githubReleases = res.body;
+          if (isStable !== undefined) {
-    const dependency: ReleaseResult = {
+            result.isStable = isStable;
-      sourceUrl: getSourceUrl(repo, registryUrl),
+          }
-      releases: githubReleases
+          return result;
-        .filter(({ draft }) => draft !== true)
+        }),
-        .map(({ tag_name, published_at, prerelease }) => ({
+      };
-          version: tag_name,
+    }
-          gitRef: tag_name,
+    return result;
-          releaseTimestamp: published_at,
-          isStable: prerelease ? false : undefined,
-        })),
-    };
-    return dependency;
   }
 }

lib/modules/datasource/github-tags/__snapshots__/index.spec.ts.snap

@@ -1,37 +0,0 @@
-// Jest Snapshot v1, https://goo.gl/fbAQLP
-
-exports[`modules/datasource/github-tags/index getReleases returns tags 1`] = `
-Object {
-  "registryUrl": "https://github.com",
-  "releases": Array [
-    Object {
-      "gitRef": "v1.0.0",
-      "releaseTimestamp": "1970-01-01T00:00:00.000Z",
-      "version": "v1.0.0",
-    },
-    Object {
-      "gitRef": "v1.1.0",
-      "isStable": false,
-      "releaseTimestamp": "1970-01-01T00:00:00.001Z",
-      "version": "v1.1.0",
-    },
-  ],
-  "sourceUrl": "https://github.com/some/dep2",
-}
-`;
-
-exports[`modules/datasource/github-tags/index getReleases supports ghe 1`] = `
-Object {
-  "releases": Array [
-    Object {
-      "gitRef": "v1.0.0",
-      "version": "v1.0.0",
-    },
-    Object {
-      "gitRef": "v1.1.0",
-      "version": "v1.1.0",
-    },
-  ],
-  "sourceUrl": "https://git.enterprise.com/some/dep2",
-}
-`;

lib/modules/datasource/github-tags/cache.spec.ts

@@ -0,0 +1,51 @@
+import { GithubHttp } from '../../../util/http/github';
+import { CacheableGithubTags, FetchedTag } from './cache';
+
+describe('modules/datasource/github-tags/cache', () => {
+  const http = new GithubHttp();
+  const cache = new CacheableGithubTags(http, { resetDeltaMinutes: 0 });
+
+  const fetchedItem: FetchedTag = {
+    version: '1.2.3',
+    target: {
+      type: 'Commit',
+      hash: 'abc',
+      releaseTimestamp: '2020-04-09T10:00:00.000Z',
+    },
+  };
+
+  describe('coerceFetched', () => {
+    it('transforms GraphQL items', () => {
+      expect(cache.coerceFetched(fetchedItem)).toEqual({
+        version: '1.2.3',
+        hash: 'abc',
+        releaseTimestamp: '2020-04-09T10:00:00.000Z',
+      });
+      expect(
+        cache.coerceFetched({
+          version: '1.2.3',
+          target: {
+            type: 'Tag',
+            target: {
+              hash: 'abc',
+              releaseTimestamp: '2020-04-09T10:00:00.000Z',
+            },
+          },
+        })
+      ).toEqual({
+        version: '1.2.3',
+        hash: 'abc',
+        releaseTimestamp: '2020-04-09T10:00:00.000Z',
+      });
+    });
+
+    it('returns null for tags we can not process', () => {
+      expect(
+        cache.coerceFetched({
+          version: '1.2.3',
+          target: { type: 'Blob' } as never,
+        })
+      ).toBeNull();
+    });
+  });
+});

lib/modules/datasource/github-tags/cache.ts

@@ -0,0 +1,88 @@
+import type { GithubHttp } from '../../../util/http/github';
+import { AbstractGithubDatasourceCache } from '../github-releases/cache/cache-base';
+import type {
+  CacheOptions,
+  StoredItemBase,
+} from '../github-releases/cache/types';
+
+const query = `
+query ($owner: String!, $name: String!, $cursor: String, $count: Int!) {
+  repository(owner: $owner, name: $name) {
+    payload: refs(
+      first: $count
+      after: $cursor
+      orderBy: {field: TAG_COMMIT_DATE, direction: DESC}
+      refPrefix: "refs/tags/"
+    ) {
+      nodes {
+        version: name
+        target {
+          type: __typename
+          ... on Commit {
+            hash: oid
+            releaseTimestamp: committedDate
+          }
+          ... on Tag {
+            target {
+              ... on Commit {
+                hash: oid
+                releaseTimestamp: committedDate
+              }
+            }
+          }
+        }
+      }
+      pageInfo {
+        hasNextPage
+        endCursor
+      }
+    }
+  }
+}
+`;
+
+export interface FetchedTag {
+  version: string;
+  target:
+    | {
+        type: 'Commit';
+        hash: string;
+        releaseTimestamp: string;
+      }
+    | {
+        type: 'Tag';
+        target: {
+          hash: string;
+          releaseTimestamp: string;
+        };
+      };
+}
+
+export interface StoredTag extends StoredItemBase {
+  hash: string;
+  releaseTimestamp: string;
+}
+
+export class CacheableGithubTags extends AbstractGithubDatasourceCache<
+  StoredTag,
+  FetchedTag
+> {
+  readonly cacheNs = 'github-datasource-graphql-tags';
+  readonly graphqlQuery = query;
+
+  constructor(http: GithubHttp, opts: CacheOptions = {}) {
+    super(http, opts);
+  }
+
+  coerceFetched(item: FetchedTag): StoredTag | null {
+    const { version, target } = item;
+    if (target.type === 'Commit') {
+      const { hash, releaseTimestamp } = target;
+      return { version, hash, releaseTimestamp };
+    } else if (target.type === 'Tag') {
+      const { hash, releaseTimestamp } = target.target;
+      return { version, hash, releaseTimestamp };
+    }
+    return null;
+  }
+}

lib/modules/datasource/github-tags/index.spec.ts

@@ -1,6 +1,8 @@
 import { getPkgReleases } from '..';
 import * as httpMock from '../../../../test/http-mock';
 import * as _hostRules from '../../../util/host-rules';
+import { CacheableGithubReleases } from '../github-releases/cache';
+import { CacheableGithubTags } from './cache';
 import { GithubTagsDatasource } from '.';
 
 jest.mock('../../../util/host-rules');
@@ -10,6 +12,15 @@ const githubApiHost = 'https://api.github.com';
 const githubEnterpriseApiHost = 'https://git.enterprise.com';
 
 describe('modules/datasource/github-tags/index', () => {
+  const releasesCacheGetItems = jest.spyOn(
+    CacheableGithubReleases.prototype,
+    'getItems'
+  );
+  const tagsCacheGetItems = jest.spyOn(
+    CacheableGithubTags.prototype,
+    'getItems'
+  );
+
   const github = new GithubTagsDatasource();
 
   beforeEach(() => {
@@ -116,37 +127,38 @@ describe('modules/datasource/github-tags/index', () => {
     const depName = 'some/dep2';
 
     it('returns tags', async () => {
-      const tags = [{ name: 'v1.0.0' }, { name: 'v1.1.0' }];
+      tagsCacheGetItems.mockResolvedValueOnce([
-      const releases = tags.map((item, idx) => ({
+        { version: 'v1.0.0', releaseTimestamp: '2021-01-01', hash: '123' },
-        tag_name: item.name,
+        { version: 'v2.0.0', releaseTimestamp: '2022-01-01', hash: 'abc' },
-        published_at: new Date(idx),
+      ]);
-        prerelease: !!idx,
+      releasesCacheGetItems.mockResolvedValueOnce([
-      }));
+        { version: 'v1.0.0', releaseTimestamp: '2021-01-01', isStable: true },
-      httpMock
+        { version: 'v2.0.0', releaseTimestamp: '2022-01-01', isStable: false },
-        .scope(githubApiHost)
+      ] as never);
-        .get(`/repos/${depName}/tags?per_page=100`)
+
-        .reply(200, tags)
-        .get(`/repos/${depName}/releases?per_page=100`)
-        .reply(200, releases);
       const res = await getPkgReleases({ datasource: github.id, depName });
-      expect(res).toMatchSnapshot();
-      expect(res.releases).toHaveLength(2);
-    });
 
-    it('supports ghe', async () => {
+      expect(res).toEqual({
-      const body = [{ name: 'v1.0.0' }, { name: 'v1.1.0' }];
+        registryUrl: 'https://github.com',
-      httpMock
+        sourceUrl: 'https://github.com/some/dep2',
-        .scope(githubEnterpriseApiHost)
+        releases: [
-        .get(`/api/v3/repos/${depName}/tags?per_page=100`)
+          {
-        .reply(200, body)
+            gitRef: 'v1.0.0',
-        .get(`/api/v3/repos/${depName}/releases?per_page=100`)
+            hash: '123',
-        .reply(404);
+            isStable: true,
+            releaseTimestamp: '2021-01-01T00:00:00.000Z',
+            version: 'v1.0.0',
+          },
 
-      const res = await github.getReleases({
+          {
-        registryUrl: 'https://git.enterprise.com',
+            gitRef: 'v2.0.0',
-        packageName: depName,
+            hash: 'abc',
+            isStable: false,
+            releaseTimestamp: '2022-01-01T00:00:00.000Z',
+            version: 'v2.0.0',
+          },
+        ],
       });
-      expect(res).toMatchSnapshot();
     });
   });
 });

lib/modules/datasource/github-tags/index.ts

@@ -8,13 +8,17 @@ import type {
   Release,
   ReleaseResult,
 } from '../types';
-import type { GitHubTag, TagResponse } from './types';
+import { CacheableGithubTags } from './cache';
+import type { TagResponse } from './types';
 
 export class GithubTagsDatasource extends GithubReleasesDatasource {
   static override readonly id = 'github-tags';
 
+  private tagsCache: CacheableGithubTags;
+
   constructor() {
     super(GithubTagsDatasource.id);
+    this.tagsCache = new CacheableGithubTags(this.http);
   }
 
   @cache({
@@ -93,45 +97,21 @@ export class GithubTagsDatasource extends GithubReleasesDatasource {
         this.getCommit(registryUrl, repo!);
   }
 
-  @cache({
-    ttlMinutes: 10,
-    namespace: `datasource-${GithubTagsDatasource.id}`,
-    key: ({ registryUrl, packageName: repo }: GetReleasesConfig) =>
-      `${registryUrl}:${repo}:tags`,
-  })
-  async getTags({
-    registryUrl,
-    packageName: repo,
-  }: GetReleasesConfig): Promise<ReleaseResult | null> {
-    const apiBaseUrl = getApiBaseUrl(registryUrl);
-    // tag
-    const url = `${apiBaseUrl}repos/${repo}/tags?per_page=100`;
-
-    const versions = (
-      await this.http.getJson<GitHubTag[]>(url, {
-        paginate: true,
-      })
-    ).body.map((o) => o.name);
-    const dependency: ReleaseResult = {
-      sourceUrl: getSourceUrl(repo, registryUrl),
-      releases: versions.map((version) => ({
-        version,
-        gitRef: version,
-      })),
-    };
-    return dependency;
-  }
-
   override async getReleases(
     config: GetReleasesConfig
   ): Promise<ReleaseResult | null> {
-    const tagsResult = await this.getTags(config);
+    const tagReleases = await this.tagsCache.getItems(config);
 
     // istanbul ignore if
-    if (!tagsResult) {
+    if (!tagReleases.length) {
       return null;
     }
 
+    const tagsResult: ReleaseResult = {
+      sourceUrl: getSourceUrl(config.packageName, config.registryUrl),
+      releases: tagReleases.map((item) => ({ ...item, gitRef: item.version })),
+    };
+
     try {
       // Fetch additional data from releases endpoint when possible
       const releasesResult = await super.getReleases(config);

lib/modules/datasource/go/__snapshots__/releases-direct.spec.ts.snap

@@ -1,21 +1,5 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`modules/datasource/go/releases-direct getReleases processes real data 1`] = `
-Object {
-  "releases": Array [
-    Object {
-      "gitRef": "v1.0.0",
-      "version": "v1.0.0",
-    },
-    Object {
-      "gitRef": "v2.0.0",
-      "version": "v2.0.0",
-    },
-  ],
-  "sourceUrl": "https://github.com/golang/text",
-}
-`;
-
 exports[`modules/datasource/go/releases-direct getReleases support bitbucket tags 1`] = `
 Object {
   "registryUrl": "https://bitbucket.org",
@@ -35,22 +19,6 @@ Object {
 }
 `;
 
-exports[`modules/datasource/go/releases-direct getReleases support ghe 1`] = `
-Object {
-  "releases": Array [
-    Object {
-      "gitRef": "v1.0.0",
-      "version": "v1.0.0",
-    },
-    Object {
-      "gitRef": "v2.0.0",
-      "version": "v2.0.0",
-    },
-  ],
-  "sourceUrl": "https://git.enterprise.com/example/module",
-}
-`;
-
 exports[`modules/datasource/go/releases-direct getReleases support gitlab 1`] = `
 Object {
   "releases": Array [

lib/modules/datasource/go/releases-direct.spec.ts

@@ -1,6 +1,7 @@
 import * as httpMock from '../../../../test/http-mock';
 import { mocked } from '../../../../test/util';
 import * as _hostRules from '../../../util/host-rules';
+import { GithubTagsDatasource } from '../github-tags';
 import { BaseGoDatasource } from './base';
 import { GoDirectDatasource } from './releases-direct';
 
@@ -12,6 +13,11 @@ const getDatasourceSpy = jest.spyOn(BaseGoDatasource, 'getDatasource');
 const hostRules = mocked(_hostRules);
 
 describe('modules/datasource/go/releases-direct', () => {
+  const githubGetTags = jest.spyOn(
+    GithubTagsDatasource.prototype,
+    'getReleases'
+  );
+
   beforeEach(() => {
     jest.resetAllMocks();
     hostRules.find.mockReturnValue({});
@@ -46,18 +52,24 @@ describe('modules/datasource/go/releases-direct', () => {
         packageName: 'golang/text',
         registryUrl: 'https://github.com',
       });
-      httpMock
+      githubGetTags.mockResolvedValueOnce({
-        .scope('https://api.github.com/')
+        releases: [
-        .get('/repos/golang/text/tags?per_page=100')
+          { gitRef: 'v1.0.0', version: 'v1.0.0' },
-        .reply(200, [{ name: 'v1.0.0' }, { name: 'v2.0.0' }])
+          { gitRef: 'v2.0.0', version: 'v2.0.0' },
-        .get('/repos/golang/text/releases?per_page=100')
+        ],
-        .reply(200, []);
+      });
+
       const res = await datasource.getReleases({
         packageName: 'golang.org/x/text',
       });
-      expect(res).toMatchSnapshot();
+
-      expect(res).not.toBeNull();
+      expect(res).toEqual({
-      expect(res).toBeDefined();
+        releases: [
+          { gitRef: 'v1.0.0', version: 'v1.0.0' },
+          { gitRef: 'v2.0.0', version: 'v2.0.0' },
+        ],
+        sourceUrl: 'https://github.com/golang/text',
+      });
     });
 
     it('support gitlab', async () => {
@@ -125,18 +137,27 @@ describe('modules/datasource/go/releases-direct', () => {
         registryUrl: 'https://git.enterprise.com',
         packageName: 'example/module',
       });
-      httpMock
+      githubGetTags.mockResolvedValueOnce({
-        .scope('https://git.enterprise.com/')
+        releases: [
-        .get('/api/v3/repos/example/module/tags?per_page=100')
+          { gitRef: 'v1.0.0', version: 'v1.0.0' },
-        .reply(200, [{ name: 'v1.0.0' }, { name: 'v2.0.0' }])
+          { gitRef: 'v2.0.0', version: 'v2.0.0' },
-        .get('/api/v3/repos/example/module/releases?per_page=100')
+        ],
-        .reply(200, []);
+      });
+
       const res = await datasource.getReleases({
         packageName: 'git.enterprise.com/example/module',
       });
-      expect(res).toMatchSnapshot();
+
-      expect(res).not.toBeNull();
+      expect(res).toEqual({
-      expect(res).toBeDefined();
+        releases: [
+          { gitRef: 'v1.0.0', version: 'v1.0.0' },
+          { gitRef: 'v2.0.0', version: 'v2.0.0' },
+        ],
+        sourceUrl: 'https://git.enterprise.com/example/module',
+      });
+      expect(githubGetTags.mock.calls).toMatchObject([
+        [{ registryUrl: 'https://git.enterprise.com' }],
+      ]);
     });
 
     it('works for known servers', async () => {
@@ -155,20 +176,7 @@ describe('modules/datasource/go/releases-direct', () => {
         packageName: 'go-x/x',
         registryUrl: 'https://github.com',
       });
-      httpMock
+      githubGetTags.mockResolvedValue({ releases: [] });
-        .scope('https://api.github.com/')
-        .get('/repos/x/text/tags?per_page=100')
-        .reply(200, [])
-        .get('/repos/x/text/releases?per_page=100')
-        .reply(200, [])
-        .get('/repos/x/text/tags?per_page=100')
-        .reply(200, [])
-        .get('/repos/x/text/releases?per_page=100')
-        .reply(200, [])
-        .get('/repos/go-x/x/tags?per_page=100')
-        .reply(200, [])
-        .get('/repos/go-x/x/releases?per_page=100')
-        .reply(200, []);
       const packages = [
         { packageName: 'github.com/x/text' },
         { packageName: 'gopkg.in/x/text' },
@@ -178,6 +186,7 @@ describe('modules/datasource/go/releases-direct', () => {
         const res = await datasource.getReleases(pkg);
         expect(res.releases).toBeEmpty();
       }
+      expect(githubGetTags).toHaveBeenCalledTimes(3);
     });
 
     it('support gitlab subgroups', async () => {
@@ -220,16 +229,15 @@ describe('modules/datasource/go/releases-direct', () => {
         { packageName: 'github.com/x/text/a' },
         { packageName: 'github.com/x/text/b' },
       ];
-      const tags = [{ name: 'a/v1.0.0' }, { name: 'b/v2.0.0' }];
+
+      githubGetTags.mockResolvedValue({
+        releases: [
+          { version: 'a/v1.0.0', gitRef: 'a/v1.0.0' },
+          { version: 'b/v2.0.0', gitRef: 'b/v2.0.0' },
+        ],
+      });
 
       for (const pkg of packages) {
-        httpMock
-          .scope('https://api.github.com/')
-          .get('/repos/x/text/tags?per_page=100')
-          .reply(200, tags)
-          .get('/repos/x/text/releases?per_page=100')
-          .reply(200, []);
-
         const prefix = pkg.packageName.split('/')[3];
         const result = await datasource.getReleases(pkg);
         expect(result.releases).toHaveLength(1);
@@ -252,16 +260,15 @@ describe('modules/datasource/go/releases-direct', () => {
         { packageName: 'github.com/x/text/a' },
         { packageName: 'github.com/x/text/b' },
       ];
-      const tags = [{ name: 'v1.0.0' }, { name: 'v2.0.0' }];
+
+      githubGetTags.mockResolvedValue({
+        releases: [
+          { version: 'v1.0.0', gitRef: 'v1.0.0' },
+          { version: 'v2.0.0', gitRef: 'v2.0.0' },
+        ],
+      });
 
       for (const pkg of packages) {
-        httpMock
-          .scope('https://api.github.com/')
-          .get('/repos/x/text/tags?per_page=100')
-          .reply(200, tags)
-          .get('/repos/x/text/releases?per_page=100')
-          .reply(200, []);
-
         const result = await datasource.getReleases(pkg);
         expect(result.releases).toHaveLength(0);
       }
@@ -274,24 +281,20 @@ describe('modules/datasource/go/releases-direct', () => {
         registryUrl: 'https://github.com',
       });
       const pkg = { packageName: 'github.com/x/text/b/v2' };
-      const tags = [
-        { name: 'a/v1.0.0' },
-        { name: 'v5.0.0' },
-        { name: 'b/v2.0.0' },
-        { name: 'b/v3.0.0' },
-      ];
 
-      httpMock
+      githubGetTags.mockResolvedValue({
-        .scope('https://api.github.com/')
+        releases: [
-        .get('/repos/x/text/tags?per_page=100')
+          { version: 'a/v1.0.0', gitRef: 'a/v1.0.0' },
-        .reply(200, tags)
+          { version: 'v5.0.0', gitRef: 'v5.0.0' },
-        .get('/repos/x/text/releases?per_page=100')
+          { version: 'b/v2.0.0', gitRef: 'b/v2.0.0' },
-        .reply(200, []);
+          { version: 'b/v3.0.0', gitRef: 'b/v3.0.0' },
+        ],
+      });
 
       const result = await datasource.getReleases(pkg);
       expect(result.releases).toEqual([
-        { gitRef: 'b/v2.0.0', version: 'v2.0.0' },
+        { version: 'v2.0.0', gitRef: 'b/v2.0.0' },
-        { gitRef: 'b/v3.0.0', version: 'v3.0.0' },
+        { version: 'v3.0.0', gitRef: 'b/v3.0.0' },
       ]);
     });
   });

lib/modules/datasource/go/releases-goproxy.spec.ts

@@ -1,10 +1,22 @@
 import * as httpMock from '../../../../test/http-mock';
 import { loadFixture } from '../../../../test/util';
+import { GithubReleasesDatasource } from '../github-releases';
+import { GithubTagsDatasource } from '../github-tags';
 import { GoProxyDatasource } from './releases-goproxy';
 
 const datasource = new GoProxyDatasource();
 
 describe('modules/datasource/go/releases-goproxy', () => {
+  const githubGetReleases = jest.spyOn(
+    GithubReleasesDatasource.prototype,
+    'getReleases'
+  );
+
+  const githubGetTags = jest.spyOn(
+    GithubTagsDatasource.prototype,
+    'getReleases'
+  );
+
   it('encodeCase', () => {
     expect(datasource.encodeCase('foo')).toBe('foo');
     expect(datasource.encodeCase('Foo')).toBe('!foo');
@@ -276,12 +288,13 @@ describe('modules/datasource/go/releases-goproxy', () => {
       process.env.GOPROXY = baseUrl;
       process.env.GOPRIVATE = 'github.com/google/*';
 
-      httpMock
+      githubGetTags.mockResolvedValueOnce({
-        .scope('https://api.github.com/')
+        releases: [
-        .get('/repos/google/btree/tags?per_page=100')
+          { gitRef: 'v1.0.0', version: 'v1.0.0' },
-        .reply(200, [{ name: 'v1.0.0' }, { name: 'v1.0.1' }])
+          { gitRef: 'v1.0.1', version: 'v1.0.1' },
-        .get('/repos/google/btree/releases?per_page=100')
+        ],
-        .reply(200, []);
+      });
+      githubGetReleases.mockResolvedValueOnce({ releases: [] });
 
       const res = await datasource.getReleases({
         packageName: 'github.com/google/btree',
@@ -458,12 +471,13 @@ describe('modules/datasource/go/releases-goproxy', () => {
         .get('/@v/list')
         .reply(410);
 
-      httpMock
+      githubGetTags.mockResolvedValueOnce({
-        .scope('https://api.github.com/')
+        releases: [
-        .get('/repos/foo/bar/tags?per_page=100')
+          { gitRef: 'v1.0.0', version: 'v1.0.0' },
-        .reply(200, [{ name: 'v1.0.0' }, { name: 'v1.0.1' }])
+          { gitRef: 'v1.0.1', version: 'v1.0.1' },
-        .get('/repos/foo/bar/releases?per_page=100')
+        ],
-        .reply(200, []);
+      });
+      githubGetReleases.mockResolvedValueOnce({ releases: [] });
 
       const res = await datasource.getReleases({
         packageName: 'github.com/foo/bar',

lib/util/http/github.ts

@@ -45,7 +45,7 @@ interface GithubGraphqlRepoData<T = unknown> {
   repository?: T;
 }
 
-interface GithubGraphqlResponse<T = unknown> {
+export interface GithubGraphqlResponse<T = unknown> {
   data?: T;
   errors?: {
     type?: string;

lib/workers/repository/process/lookup/index.spec.ts

@@ -19,22 +19,6 @@ import * as lookup from '.';
 
 jest.mock('../../../../modules/datasource/docker');
 
-jest.mock('../../../../modules/datasource/git-refs', function () {
-  const { GitRefsDatasource: Orig } = jest.requireActual(
-    '../../../../modules/datasource/git-refs'
-  );
-  const Mocked = jest.fn().mockImplementation(() => ({
-    getReleases: () =>
-      Promise.resolve({
-        releases: [{ version: 'master' }],
-      }),
-    getDigest: () =>
-      Promise.resolve('4b825dc642cb6eb9a060e54bf8d69288fbee4904'),
-  }));
-  Mocked['id'] = Orig.id;
-  return { GitRefsDatasource: Mocked };
-});
-
 const fixtureRoot = '../../../../config/npm';
 const qJson = {
   ...Fixtures.getJson('01.json', fixtureRoot),
@@ -53,6 +37,11 @@ const docker = mocked(DockerDatasource.prototype);
 let config: LookupUpdateConfig;
 
 describe('workers/repository/process/lookup/index', () => {
+  const getGithubReleases = jest.spyOn(
+    GithubReleasesDatasource.prototype,
+    'getReleases'
+  );
+
   beforeEach(() => {
     // TODO: fix types #7154
     config = partial<LookupUpdateConfig>(getConfig() as never);
@@ -60,6 +49,14 @@ describe('workers/repository/process/lookup/index', () => {
     config.versioning = npmVersioningId;
     config.rangeStrategy = 'replace';
     jest.resetAllMocks();
+    jest
+      .spyOn(GitRefsDatasource.prototype, 'getReleases')
+      .mockResolvedValueOnce({
+        releases: [{ version: 'master' }],
+      });
+    jest
+      .spyOn(GitRefsDatasource.prototype, 'getDigest')
+      .mockResolvedValueOnce('4b825dc642cb6eb9a060e54bf8d69288fbee4904');
   });
 
   // TODO: fix mocks
@@ -884,14 +881,13 @@ describe('workers/repository/process/lookup/index', () => {
       config.currentValue = '1.4.4';
       config.depName = 'some/action';
       config.datasource = GithubReleasesDatasource.id;
-      httpMock
+      getGithubReleases.mockResolvedValueOnce({
-        .scope('https://api.github.com')
+        releases: [
-        .get('/repos/some/action/releases?per_page=100')
+          { version: '1.4.4' },
-        .reply(200, [
+          { version: '2.0.0' },
-          { tag_name: '1.4.4' },
+          { version: '2.1.0', isStable: false },
-          { tag_name: '2.0.0' },
+        ],
-          { tag_name: '2.1.0', prerelease: true },
+      });
-        ]);
       expect((await lookup.lookupUpdates(config)).updates).toMatchSnapshot([
         { newValue: '2.0.0', updateType: 'major' },
       ]);
@@ -907,14 +903,13 @@ describe('workers/repository/process/lookup/index', () => {
       yesterday.setDate(yesterday.getDate() - 1);
       const lastWeek = new Date();
       lastWeek.setDate(lastWeek.getDate() - 7);
-      httpMock
+      getGithubReleases.mockResolvedValueOnce({
-        .scope('https://api.github.com')
+        releases: [
-        .get('/repos/some/action/releases?per_page=100')
+          { version: '1.4.4' },
-        .reply(200, [
+          { version: '1.4.5', releaseTimestamp: lastWeek.toISOString() },
-          { tag_name: '1.4.4' },
+          { version: '1.4.6', releaseTimestamp: yesterday.toISOString() },
-          { tag_name: '1.4.5', published_at: lastWeek.toISOString() },
+        ],
-          { tag_name: '1.4.6', published_at: yesterday.toISOString() },
+      });
-        ]);
       const res = await lookup.lookupUpdates(config);
       expect(res.updates).toHaveLength(1);
       expect(res.updates[0].newVersion).toBe('1.4.6');
@@ -931,14 +926,13 @@ describe('workers/repository/process/lookup/index', () => {
       yesterday.setDate(yesterday.getDate() - 1);
       const lastWeek = new Date();
       lastWeek.setDate(lastWeek.getDate() - 7);
-      httpMock
+      getGithubReleases.mockResolvedValueOnce({
-        .scope('https://api.github.com')
+        releases: [
-        .get('/repos/some/action/releases?per_page=100')
+          { version: '1.4.4' },
-        .reply(200, [
+          { version: '1.4.5', releaseTimestamp: lastWeek.toISOString() },
-          { tag_name: '1.4.4' },
+          { version: '1.4.6', releaseTimestamp: yesterday.toISOString() },
-          { tag_name: '1.4.5', published_at: lastWeek.toISOString() },
+        ],
-          { tag_name: '1.4.6', published_at: yesterday.toISOString() },
+      });
-        ]);
       const res = await lookup.lookupUpdates(config);
       expect(res.updates).toHaveLength(1);
       expect(res.updates[0].newVersion).toBe('1.4.5');

lib/workers/repository/update/pr/changelog/github.spec.ts

@@ -1,6 +1,7 @@
 import * as httpMock from '../../../../../../test/http-mock';
 import { GlobalConfig } from '../../../../../config/global';
 import { PlatformId } from '../../../../../constants';
+import { CacheableGithubTags } from '../../../../../modules/datasource/github-tags/cache';
 import * as semverVersioning from '../../../../../modules/versioning/semver';
 import * as hostRules from '../../../../../util/host-rules';
 import type { BranchUpgradeConfig } from '../../../../types';
@@ -35,6 +36,7 @@ describe('workers/repository/update/pr/changelog/github', () => {
   afterEach(() => {
     // FIXME: add missing http mocks
     httpMock.clear(false);
+    jest.resetAllMocks();
   });
 
   describe('getChangeLogJSON', () => {
@@ -297,15 +299,17 @@ describe('workers/repository/update/pr/changelog/github', () => {
     });
 
     it('works with same version releases but different prefix', async () => {
-      httpMock
+      const githubTagsMock = jest.spyOn(
-        .scope('https://api.github.com/')
+        CacheableGithubTags.prototype,
-        .get('/repos/chalk/chalk/tags?per_page=100')
+        'getItems'
-        .reply(200, [
+      );
-          { name: 'v1.0.1' },
+
-          { name: '1.0.1' },
+      githubTagsMock.mockResolvedValue([
-          { name: 'correctPrefix/target@1.0.1' },
+        { version: 'v1.0.1' },
-          { name: 'wrongPrefix/target-1.0.1' },
+        { version: '1.0.1' },
-        ]);
+        { version: 'correctPrefix/target@1.0.1' },
+        { version: 'wrongPrefix/target-1.0.1' },
+      ] as never);
 
       const upgradeData: BranchUpgradeConfig = {
         manager: 'some-manager',

lib/workers/repository/update/pr/changelog/github/index.ts

@@ -1,7 +1,7 @@
 import changelogFilenameRegex from 'changelog-filename-regex';
 import { logger } from '../../../../../../logger';
-import type { GithubRelease } from '../../../../../../modules/datasource/github-releases/types';
+import { CacheableGithubReleases } from '../../../../../../modules/datasource/github-releases/cache';
-import type { GitHubTag } from '../../../../../../modules/datasource/github-tags/types';
+import { CacheableGithubTags } from '../../../../../../modules/datasource/github-tags/cache';
 import type {
   GithubGitBlob,
   GithubGitTree,
@@ -14,25 +14,26 @@ import type { ChangeLogFile, ChangeLogNotes } from '../types';
 
 export const id = 'github-changelog';
 const http = new GithubHttp(id);
+const tagsCache = new CacheableGithubTags(http);
+const releasesCache = new CacheableGithubReleases(http);
 
 export async function getTags(
   endpoint: string,
   repository: string
 ): Promise<string[]> {
   logger.trace('github.getTags()');
-  const url = `${endpoint}repos/${repository}/tags?per_page=100`;
   try {
-    const res = await http.getJson<GitHubTag[]>(url, {
+    const tags = await tagsCache.getItems({
-      paginate: true,
+      registryUrl: endpoint,
+      packageName: repository,
     });
 
-    const tags = res.body;
+    // istanbul ignore if
-
     if (!tags.length) {
       logger.debug({ repository }, 'repository has no Github tags');
     }
 
-    return tags.map((tag) => tag.name).filter(Boolean);
+    return tags.map(({ version }) => version).filter(Boolean);
   } catch (err) {
     logger.debug(
       { sourceRepo: repository, err },
@@ -110,16 +111,19 @@ export async function getReleaseList(
   repository: string
 ): Promise<ChangeLogNotes[]> {
   logger.trace('github.getReleaseList()');
-  const url = `${ensureTrailingSlash(apiBaseUrl)}repos/${repository}/releases`;
+  const notesSourceUrl = `${ensureTrailingSlash(
-  const res = await http.getJson<GithubRelease[]>(`${url}?per_page=100`, {
+    apiBaseUrl
-    paginate: true,
+  )}repos/${repository}/releases`;
+  const items = await releasesCache.getItems({
+    registryUrl: apiBaseUrl,
+    packageName: repository,
   });
-  return res.body.map((release) => ({
+  return items.map(({ url, id, version: tag, name, description: body }) => ({
-    url: release.html_url,
+    url,
-    notesSourceUrl: url,
+    notesSourceUrl,
-    id: release.id,
+    id,
-    tag: release.tag_name,
+    tag,
-    name: release.name,
+    name,
-    body: release.body,
+    body,
   }));
 }

lib/workers/repository/update/pr/changelog/index.spec.ts

@@ -2,6 +2,8 @@ import * as httpMock from '../../../../../../test/http-mock';
 import { partial } from '../../../../../../test/util';
 import { GlobalConfig } from '../../../../../config/global';
 import { PlatformId } from '../../../../../constants';
+import { CacheableGithubReleases } from '../../../../../modules/datasource/github-releases/cache';
+import { CacheableGithubTags } from '../../../../../modules/datasource/github-tags/cache';
 import * as semverVersioning from '../../../../../modules/versioning/semver';
 import * as hostRules from '../../../../../util/host-rules';
 import type { BranchConfig } from '../../../../types';
@@ -34,7 +36,17 @@ const upgrade: BranchConfig = partial<BranchConfig>({
 
 describe('workers/repository/update/pr/changelog/index', () => {
   describe('getChangeLogJSON', () => {
+    const githubReleasesMock = jest.spyOn(
+      CacheableGithubReleases.prototype,
+      'getItems'
+    );
+    const githubTagsMock = jest.spyOn(
+      CacheableGithubTags.prototype,
+      'getItems'
+    );
+
     beforeEach(() => {
+      jest.resetAllMocks();
       hostRules.clear();
       hostRules.add({
         hostType: PlatformId.Github,
@@ -81,15 +93,12 @@ describe('workers/repository/update/pr/changelog/index', () => {
     });
 
     it('works without Github', async () => {
+      githubTagsMock.mockRejectedValueOnce(new Error('Unknown'));
+      githubReleasesMock.mockRejectedValueOnce(new Error('Unknown'));
       httpMock
         .scope(githubApiHost)
         .get('/repos/chalk/chalk')
         .times(4)
-        .reply(500)
-        .get('/repos/chalk/chalk/tags?per_page=100')
-        .reply(500)
-        .get('/repos/chalk/chalk/releases?per_page=100')
-        .times(4)
         .reply(500);
       expect(
         await getChangeLogJSON({
@@ -116,20 +125,16 @@ describe('workers/repository/update/pr/changelog/index', () => {
     });
 
     it('uses GitHub tags', async () => {
-      httpMock
+      httpMock.scope(githubApiHost).get(/.*/).reply(200, []).persist();
-        .scope(githubApiHost)
+      githubTagsMock.mockResolvedValue([
-        .get('/repos/chalk/chalk/tags?per_page=100')
+        { version: '0.9.0' },
-        .reply(200, [
+        { version: '1.0.0' },
-          { name: '0.9.0' },
+        { version: '1.4.0' },
-          { name: '1.0.0' },
+        { version: 'v2.3.0' },
-          { name: '1.4.0' },
+        { version: '2.2.2' },
-          { name: 'v2.3.0' },
+        { version: 'v2.4.2' },
-          { name: '2.2.2' },
+      ] as never);
-          { name: 'v2.4.2' },
+      githubReleasesMock.mockResolvedValue([]);
-        ])
-        .persist()
-        .get(/.*/)
-        .reply(200, []);
       expect(
         await getChangeLogJSON({
           ...upgrade,
@@ -155,11 +160,11 @@ describe('workers/repository/update/pr/changelog/index', () => {
     });
 
     it('filters unnecessary warns', async () => {
-      httpMock
+      githubTagsMock.mockRejectedValueOnce(new Error('Unknown Github Repo'));
-        .scope(githubApiHost)
+      githubReleasesMock.mockRejectedValueOnce(
-        .persist()
+        new Error('Unknown Github Repo')
-        .get(/.*/)
+      );
-        .replyWithError('Unknown Github Repo');
+      httpMock.scope(githubApiHost).get(/.*/).reply(200, []).persist();
       const res = await getChangeLogJSON({
         ...upgrade,
         depName: '@renovate/no',
@@ -185,6 +190,8 @@ describe('workers/repository/update/pr/changelog/index', () => {
     });
 
     it('supports node engines', async () => {
+      githubTagsMock.mockRejectedValueOnce([]);
+      githubReleasesMock.mockRejectedValueOnce([]);
       expect(
         await getChangeLogJSON({
           ...upgrade,
@@ -259,6 +266,8 @@ describe('workers/repository/update/pr/changelog/index', () => {
     });
 
     it('supports github enterprise and github.com changelog', async () => {
+      githubTagsMock.mockRejectedValueOnce([]);
+      githubReleasesMock.mockRejectedValueOnce([]);
       httpMock.scope(githubApiHost).persist().get(/.*/).reply(200, []);
       hostRules.add({
         hostType: PlatformId.Github,
@@ -291,6 +300,8 @@ describe('workers/repository/update/pr/changelog/index', () => {
     });
 
     it('supports github enterprise and github enterprise changelog', async () => {
+      githubTagsMock.mockRejectedValueOnce([]);
+      githubReleasesMock.mockRejectedValueOnce([]);
       httpMock
         .scope('https://github-enterprise.example.com')
         .persist()
@@ -329,6 +340,8 @@ describe('workers/repository/update/pr/changelog/index', () => {
     });
 
     it('supports github.com and github enterprise changelog', async () => {
+      githubTagsMock.mockRejectedValueOnce([]);
+      githubReleasesMock.mockRejectedValueOnce([]);
       httpMock
         .scope('https://github-enterprise.example.com')
         .persist()

lib/workers/repository/update/pr/changelog/release-notes.spec.ts

@@ -1,6 +1,7 @@
 import { DateTime } from 'luxon';
 import * as httpMock from '../../../../../../test/http-mock';
 import { loadFixture, mocked } from '../../../../../../test/util';
+import { CacheableGithubReleases } from '../../../../../modules/datasource/github-releases/cache';
 import { clone } from '../../../../../util/clone';
 import * as _hostRules from '../../../../../util/host-rules';
 import { toBase64 } from '../../../../../util/string';
@@ -56,6 +57,11 @@ const gitlabProject = {
 } as ChangeLogProject;
 
 describe('workers/repository/update/pr/changelog/release-notes', () => {
+  const githubReleasesMock = jest.spyOn(
+    CacheableGithubReleases.prototype,
+    'getItems'
+  );
+
   beforeEach(() => {
     hostRules.find.mockReturnValue({});
     hostRules.hosts.mockReturnValue([]);
@@ -161,16 +167,14 @@ describe('workers/repository/update/pr/changelog/release-notes', () => {
     });
 
     it('should return release list for github repo', async () => {
-      httpMock
+      githubReleasesMock.mockResolvedValueOnce([
-        .scope('https://api.github.com/')
+        { version: `v1.0.0` },
-        .get('/repos/some/yet-other-repository/releases?per_page=100')
+        {
-        .reply(200, [
+          version: `v1.0.1`,
-          { tag_name: `v1.0.0` },
+          description:
-          {
+            'some body #123, [#124](https://github.com/some/yet-other-repository/issues/124)',
-            tag_name: `v1.0.1`,
+        },
-            body: 'some body #123, [#124](https://github.com/some/yet-other-repository/issues/124)',
+      ] as never);
-          },
-        ]);
 
       const res = await getReleaseList({
         ...githubProject,
@@ -263,10 +267,10 @@ describe('workers/repository/update/pr/changelog/release-notes', () => {
 
   describe('getReleaseNotes()', () => {
     it('should return null for release notes without body', async () => {
-      httpMock
+      githubReleasesMock.mockResolvedValueOnce([
-        .scope('https://api.github.com/')
+        { version: 'v1.0.0' },
-        .get('/repos/some/repository/releases?per_page=100')
+        { version: 'v1.0.1' },
-        .reply(200, [{ tag_name: 'v1.0.0' }, { tag_name: 'v1.0.1' }]);
+      ] as never);
       const res = await getReleaseNotes(
         {
           ...githubProject,
@@ -279,17 +283,14 @@ describe('workers/repository/update/pr/changelog/release-notes', () => {
     });
 
     it('gets release notes with body ""', async () => {
-      const prefix = '';
+      githubReleasesMock.mockResolvedValueOnce([
-      httpMock
+        { version: '1.0.0' },
-        .scope('https://api.github.com/')
+        {
-        .get('/repos/some/other-repository/releases?per_page=100')
+          version: '1.0.1',
-        .reply(200, [
+          description:
-          { tag_name: `${prefix}1.0.0` },
+            'some body #123, [#124](https://github.com/some/yet-other-repository/issues/124)',
-          {
+        },
-            tag_name: `${prefix}1.0.1`,
+      ] as never);
-            body: 'some body #123, [#124](https://github.com/some/yet-other-repository/issues/124)',
-          },
-        ]);
       const res = await getReleaseNotes(
         {
           ...githubProject,
@@ -310,17 +311,14 @@ describe('workers/repository/update/pr/changelog/release-notes', () => {
     });
 
     it('gets release notes with body "v"', async () => {
-      const prefix = 'v';
+      githubReleasesMock.mockResolvedValueOnce([
-      httpMock
+        { version: 'v1.0.0' },
-        .scope('https://api.github.com/')
+        {
-        .get('/repos/some/other-repository/releases?per_page=100')
+          version: 'v1.0.1',
-        .reply(200, [
+          description:
-          { tag_name: `${prefix}1.0.0` },
+            'some body #123, [#124](https://github.com/some/yet-other-repository/issues/124)',
-          {
+        },
-            tag_name: `${prefix}1.0.1`,
+      ] as never);
-            body: 'some body #123, [#124](https://github.com/some/yet-other-repository/issues/124)',
-          },
-        ]);
       const res = await getReleaseNotes(
         {
           ...githubProject,
@@ -341,17 +339,15 @@ describe('workers/repository/update/pr/changelog/release-notes', () => {
     });
 
     it('gets release notes with body "other-"', async () => {
-      const prefix = 'other-';
+      githubReleasesMock.mockResolvedValueOnce([
-      httpMock
+        { version: 'other-1.0.0' },
-        .scope('https://api.github.com/')
+        {
-        .get('/repos/some/other-repository/releases?per_page=100')
+          version: 'other-1.0.1',
-        .reply(200, [
+          description:
-          { tag_name: `${prefix}1.0.0` },
+            'some body #123, [#124](https://github.com/some/yet-other-repository/issues/124)',
-          {
+        },
-            tag_name: `${prefix}1.0.1`,
+      ] as never);
-            body: 'some body #123, [#124](https://github.com/some/yet-other-repository/issues/124)',
+
-          },
-        ]);
       const res = await getReleaseNotes(
         {
           ...githubProject,
@@ -372,17 +368,15 @@ describe('workers/repository/update/pr/changelog/release-notes', () => {
     });
 
     it('gets release notes with body "other_v"', async () => {
-      const prefix = 'other_v';
+      githubReleasesMock.mockResolvedValueOnce([
-      httpMock
+        { version: 'other_v1.0.0' },
-        .scope('https://api.github.com/')
+        {
-        .get('/repos/some/other-repository/releases?per_page=100')
+          version: 'other_v1.0.1',
-        .reply(200, [
+          description:
-          { tag_name: `${prefix}1.0.0` },
+            'some body #123, [#124](https://github.com/some/yet-other-repository/issues/124)',
-          {
+        },
-            tag_name: `${prefix}1.0.1`,
+      ] as never);
-            body: 'some body #123, [#124](https://github.com/some/yet-other-repository/issues/124)',
+
-          },
-        ]);
       const res = await getReleaseNotes(
         {
           ...githubProject,
@@ -403,17 +397,14 @@ describe('workers/repository/update/pr/changelog/release-notes', () => {
     });
 
     it('gets release notes with body "other@"', async () => {
-      const prefix = 'other@';
+      githubReleasesMock.mockResolvedValueOnce([
-      httpMock
+        { version: 'other@1.0.0' },
-        .scope('https://api.github.com/')
+        {
-        .get('/repos/some/other-repository/releases?per_page=100')
+          version: 'other@1.0.1',
-        .reply(200, [
+          description:
-          { tag_name: `${prefix}1.0.0` },
+            'some body #123, [#124](https://github.com/some/yet-other-repository/issues/124)',
-          {
+        },
-            tag_name: `${prefix}1.0.1`,
+      ] as never);
-            body: 'some body #123, [#124](https://github.com/some/yet-other-repository/issues/124)',
-          },
-        ]);
       const res = await getReleaseNotes(
         {
           ...githubProject,
@@ -547,20 +538,17 @@ describe('workers/repository/update/pr/changelog/release-notes', () => {
 
     it('handles same version but different repo releases', async () => {
       const depName = 'correctTagPrefix/exampleDep';
-      httpMock
+      githubReleasesMock.mockResolvedValueOnce([
-        .scope('https://api.github.com/')
+        {
-        .get('/repos/some/other-repository/releases?per_page=100')
+          version: `${depName}@1.0.0`,
-        .reply(200, [
+          url: 'correct/url/tag.com',
-          {
+          description: 'some body',
-            tag_name: `${depName}@1.0.0`,
+        },
-            html_url: 'correct/url/tag.com',
+        { version: `someOtherRelease1/exampleDep_1.0.0` },
-            body: 'some body',
+        {
-          },
+          version: `someOtherRelease2/exampleDep-1.0.0`,
-          { tag_name: `someOtherRelease1/exampleDep_1.0.0` },
+        },
-          {
+      ] as never);
-            tag_name: `someOtherRelease2/exampleDep-1.0.0`,
-          },
-        ]);
       const res = await getReleaseNotes(
         {
           ...githubProject,