archive/crazy-max_diun
0
0
Fork 0
mirror of https://github.com/crazy-max/diun.git synced 2025-04-18 00:42:35 +00:00
Code Issues Projects Releases Wiki Activity

Implement Swarm provider

Browse source
This commit is contained in:
CrazyMax 2019-12-14 03:55:58 +01:00
parent 827703aa72
commit 629f98af4e
No known key found for this signature in database
GPG key ID: 3248E46B6BB8C7F7
12 changed files with 245 additions and 103 deletions

6
internal/app/diun.go
View file

@ -11,6 +11,7 @@ import (
"github.com/crazy-max/diun/internal/notif"
dockerPrd "github.com/crazy-max/diun/internal/provider/docker"
imagePrd "github.com/crazy-max/diun/internal/provider/image"
swarmPrd "github.com/crazy-max/diun/internal/provider/swarm"
"github.com/hako/durafmt"
"github.com/panjf2000/ants/v2"
"github.com/robfig/cron/v3"
@ -108,6 +109,11 @@ func (di *Diun) Run() {
di.createJob(job)
}
// Swarm provider
for _, job := range swarmPrd.New(di.cfg.Providers.Swarm).ListJob() {
di.createJob(job)
}
// Image provider
for _, job := range imagePrd.New(di.cfg.Providers.Image).ListJob() {
di.createJob(job)

50
internal/config/config.go
View file

@ -63,6 +63,7 @@ func Load(flags model.Flags, version string) (*Config, error) {
},
Providers: model.Providers{
Docker: []model.PrdDocker{},
Swarm: []model.PrdSwarm{},
Image: []model.PrdImage{},
},
}
@ -100,14 +101,20 @@ func (cfg *Config) validate() error {
}
}
for key, dock := range cfg.Providers.Docker {
if err := cfg.validateDockerProvider(key, dock); err != nil {
for key, prdDocker := range cfg.Providers.Docker {
if err := cfg.validateDockerProvider(key, prdDocker); err != nil {
return err
}
}
for key, img := range cfg.Providers.Image {
if err := cfg.validateImageProvider(key, img); err != nil {
for key, prdSwarm := range cfg.Providers.Swarm {
if err := cfg.validateSwarmProvider(key, prdSwarm); err != nil {
return err
}
}
for key, prdImage := range cfg.Providers.Image {
if err := cfg.validateImageProvider(key, prdImage); err != nil {
return err
}
}
@ -134,37 +141,52 @@ func (cfg *Config) validateRegOpts(id string, regopts model.RegOpts) error {
InsecureTLS: false,
Timeout: defTimeout,
}); err != nil {
return fmt.Errorf("cannot set default registry options values for %s: %v", id, err)
return fmt.Errorf("cannot set default values for registry options %s: %v", id, err)
}
cfg.RegOpts[id] = regopts
return nil
}
func (cfg *Config) validateDockerProvider(key int, dock model.PrdDocker) error {
if dock.ID == "" {
func (cfg *Config) validateDockerProvider(key int, prdDocker model.PrdDocker) error {
if prdDocker.ID == "" {
return fmt.Errorf("id is required for docker provider %d", key)
}
if err := mergo.Merge(&dock, model.PrdDocker{
if err := mergo.Merge(&prdDocker, model.PrdDocker{
TLSVerify: true,
SwarmMode: false,
WatchByDefault: false,
WatchStopped: false,
}); err != nil {
return fmt.Errorf("cannot set default docker provider values for %s: %v", dock.ID, err)
return fmt.Errorf("cannot set default values for docker provider %s: %v", prdDocker.ID, err)
}
cfg.Providers.Docker[key] = dock
cfg.Providers.Docker[key] = prdDocker
return nil
}
func (cfg *Config) validateImageProvider(key int, img model.PrdImage) error {
if img.Name == "" {
func (cfg *Config) validateSwarmProvider(key int, prdSwarm model.PrdSwarm) error {
if prdSwarm.ID == "" {
return fmt.Errorf("id is required for swarm provider %d", key)
}
if err := mergo.Merge(&prdSwarm, model.PrdSwarm{
TLSVerify: true,
WatchByDefault: false,
}); err != nil {
return fmt.Errorf("cannot set default values for swarm provider %s: %v", prdSwarm.ID, err)
}
cfg.Providers.Swarm[key] = prdSwarm
return nil
}
func (cfg *Config) validateImageProvider(key int, prdImage model.PrdImage) error {
if prdImage.Name == "" {
return fmt.Errorf("name is required for image provider %d", key)
}
cfg.Providers.Image[key] = img
cfg.Providers.Image[key] = prdImage
return nil
}

6
internal/config/config.test.yml
View file

@ -39,7 +39,11 @@ regopts:
providers:
docker:
- id: local
- id: standalone
watch_by_default: true
watch_stopped: true
swarm:
- id: local_swarm
watch_by_default: true
image:
- name: docker.io/crazymax/nextcloud:latest

33
internal/config/config_test.go
View file

@ -86,21 +86,26 @@ func TestLoad(t *testing.T) {
Providers: model.Providers{
Docker: []model.PrdDocker{
{
ID: "local",
ID: "standalone",
TLSVerify: true,
WatchByDefault: true,
WatchStopped: true,
},
},
Swarm: []model.PrdSwarm{
{
ID: "local_swarm",
TLSVerify: true,
WatchByDefault: true,
},
},
Image: []model.PrdImage{
{
Name: "docker.io/crazymax/nextcloud:latest",
Os: "linux",
Arch: "amd64",
RegOptsID: "someregopts",
},
{
Name: "crazymax/swarm-cronjob",
Os: "linux",
Arch: "amd64",
WatchRepo: true,
IncludeTags: []string{
`^1\.2\..*`,
@ -108,26 +113,18 @@ func TestLoad(t *testing.T) {
},
{
Name: "jfrog-docker-reg2.bintray.io/jfrog/artifactory-oss:4.0.0",
Os: "linux",
Arch: "amd64",
RegOptsID: "bintrayoptions",
},
{
Name: "docker.bintray.io/jfrog/xray-server:2.8.6",
Os: "linux",
Arch: "amd64",
WatchRepo: true,
MaxTags: 50,
},
{
Name: "quay.io/coreos/hyperkube",
Os: "linux",
Arch: "amd64",
},
{
Name: "docker.io/portainer/portainer",
Os: "linux",
Arch: "amd64",
WatchRepo: true,
MaxTags: 10,
IncludeTags: []string{
@ -136,8 +133,6 @@ func TestLoad(t *testing.T) {
},
{
Name: "traefik",
Os: "linux",
Arch: "amd64",
WatchRepo: true,
},
{
@ -147,23 +142,15 @@ func TestLoad(t *testing.T) {
},
{
Name: "docker.io/graylog/graylog:3.2.0",
Os: "linux",
Arch: "amd64",
},
{
Name: "jacobalberty/unifi:5.9",
Os: "linux",
Arch: "amd64",
},
{
Name: "quay.io/coreos/hyperkube:v1.1.7-coreos.1",
Os: "linux",
Arch: "amd64",
},
{
Name: "crazymax/ddns-route53",
Os: "linux",
Arch: "amd64",
WatchRepo: true,
IncludeTags: []string{
`^1\..*`,

20
internal/model/providers.go
View file

@ -2,13 +2,11 @@ package model
// Providers represents a provider configuration
type Providers struct {
Image []PrdImage `yaml:"image,omitempty" json:",omitempty"`
Docker []PrdDocker `yaml:"docker,omitempty" json:",omitempty"`
Swarm []PrdSwarm `yaml:"swarm,omitempty" json:",omitempty"`
Image []PrdImage `yaml:"image,omitempty" json:",omitempty"`
}
// PrdImage holds image provider configuration
type PrdImage Image
// PrdDocker holds docker provider configuration
type PrdDocker struct {
ID string `yaml:"id,omitempty" json:",omitempty"`
@ -16,7 +14,19 @@ type PrdDocker struct {
ApiVersion string `yaml:"api_version,omitempty" json:",omitempty"`
TLSCertsPath string `yaml:"tls_certs_path,omitempty" json:",omitempty"`
TLSVerify bool `yaml:"tls_verify,omitempty" json:",omitempty"`
SwarmMode bool `yaml:"swarm_mode,omitempty" json:",omitempty"`
WatchByDefault bool `yaml:"watch_by_default,omitempty" json:",omitempty"`
WatchStopped bool `yaml:"watch_stopped,omitempty" json:",omitempty"`
}
// PrdSwarm holds swarm provider configuration
type PrdSwarm struct {
ID string `yaml:"id,omitempty" json:",omitempty"`
Endpoint string `yaml:"endpoint,omitempty" json:",omitempty"`
ApiVersion string `yaml:"api_version,omitempty" json:",omitempty"`
TLSCertsPath string `yaml:"tls_certs_path,omitempty" json:",omitempty"`
TLSVerify bool `yaml:"tls_verify,omitempty" json:",omitempty"`
WatchByDefault bool `yaml:"watch_by_default,omitempty" json:",omitempty"`
}
// PrdImage holds image provider configuration
type PrdImage Image

55
internal/provider/common.go Normal file
View file

@ -0,0 +1,55 @@
package provider
import (
"fmt"
"strconv"
"strings"
"github.com/crazy-max/diun/internal/model"
)
func ValidateContainerImage(image string, labels map[string]string, watchByDef bool) (img model.Image, err error) {
if i := strings.Index(image, "@sha256:"); i > 0 {
image = image[:i]
}
img = model.Image{
Name: image,
}
if enableStr, ok := labels["diun.enable"]; ok {
enable, err := strconv.ParseBool(enableStr)
if err != nil {
return img, fmt.Errorf("cannot parse %s value of label diun.enable", enableStr)
}
if !enable {
return model.Image{}, nil
}
} else if !watchByDef {
return model.Image{}, nil
}
for key, value := range labels {
switch key {
case "diun.os":
img.Os = value
case "diun.arch":
img.Arch = value
case "diun.regopts_id":
img.RegOptsID = value
case "diun.watch_repo":
if img.WatchRepo, err = strconv.ParseBool(value); err != nil {
return img, fmt.Errorf("cannot parse %s value of label %s", value, key)
}
case "diun.max_tags":
if img.MaxTags, err = strconv.Atoi(value); err != nil {
return img, fmt.Errorf("cannot parse %s value of label %s", value, key)
}
case "diun.include_tags":
img.IncludeTags = strings.Split(value, ";")
case "diun.exclude_tags":
img.ExcludeTags = strings.Split(value, ";")
}
}
return img, nil
}

54
internal/provider/docker/container.go
View file

@ -1,14 +1,11 @@
package docker
import (
"fmt"
"reflect"
"strconv"
"strings"
"github.com/crazy-max/diun/internal/model"
"github.com/crazy-max/diun/internal/provider"
"github.com/crazy-max/diun/pkg/docker"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/rs/zerolog/log"
)
@ -32,7 +29,7 @@ func (c *Client) listContainerImage(elt model.PrdDocker) []model.Image {
ctnFilter.Add("status", "exited")
}
ctns, err := cli.Containers(ctnFilter)
ctns, err := cli.ContainerList(ctnFilter)
if err != nil {
sublog.Error().Err(err).Msg("Cannot list Docker containers")
return []model.Image{}
@ -40,9 +37,9 @@ func (c *Client) listContainerImage(elt model.PrdDocker) []model.Image {
var list []model.Image
for _, ctn := range ctns {
image, err := c.containerImage(elt, ctn)
image, err := provider.ValidateContainerImage(ctn.Image, ctn.Labels, elt.WatchByDefault)
if err != nil {
sublog.Error().Err(err).Msgf("Cannot get image for container %s", ctn.ID)
sublog.Error().Err(err).Msgf("Cannot get image from container %s", ctn.ID)
continue
} else if reflect.DeepEqual(image, model.Image{}) {
sublog.Debug().Msgf("Watch disabled for container %s", ctn.ID)
@ -53,46 +50,3 @@ func (c *Client) listContainerImage(elt model.PrdDocker) []model.Image {
return list
}
func (c *Client) containerImage(elt model.PrdDocker, ctn types.Container) (img model.Image, err error) {
img = model.Image{
Name: ctn.Image,
}
if enableStr, ok := ctn.Labels["diun.enable"]; ok {
enable, err := strconv.ParseBool(enableStr)
if err != nil {
return img, fmt.Errorf("cannot parse %s value of label diun.enable", enableStr)
}
if !enable {
return model.Image{}, nil
}
} else if !elt.WatchByDefault {
return model.Image{}, nil
}
for key, value := range ctn.Labels {
switch key {
case "diun.os":
img.Os = value
case "diun.arch":
img.Arch = value
case "diun.regopts_id":
img.RegOptsID = value
case "diun.watch_repo":
if img.WatchRepo, err = strconv.ParseBool(value); err != nil {
return img, fmt.Errorf("cannot parse %s value of label %s", value, key)
}
case "diun.max_tags":
if img.MaxTags, err = strconv.Atoi(value); err != nil {
return img, fmt.Errorf("cannot parse %s value of label %s", value, key)
}
case "diun.include_tags":
img.IncludeTags = strings.Split(value, ";")
case "diun.exclude_tags":
img.ExcludeTags = strings.Split(value, ";")
}
}
return img, nil
}

6
internal/provider/docker/docker.go
View file

@ -28,12 +28,6 @@ func (c *Client) ListJob() []model.Job {
log.Info().Msgf("Found %d docker provider(s) to analyze...", len(c.elts))
var list []model.Job
for _, elt := range c.elts {
// Swarm mode
if elt.SwarmMode {
continue
}
// Docker
for _, img := range c.listContainerImage(elt) {
list = append(list, model.Job{
Provider: "docker",

45
internal/provider/swarm/service.go Normal file
View file

@ -0,0 +1,45 @@
package swarm
import (
"reflect"
"github.com/crazy-max/diun/internal/model"
"github.com/crazy-max/diun/internal/provider"
"github.com/crazy-max/diun/pkg/docker"
"github.com/docker/docker/api/types/filters"
"github.com/rs/zerolog/log"
)
func (c *Client) listServiceImage(elt model.PrdSwarm) []model.Image {
sublog := log.With().
Str("provider", "swarm").
Str("id", elt.ID).
Logger()
cli, err := docker.NewClient(elt.Endpoint, elt.ApiVersion, elt.TLSCertsPath, elt.TLSVerify)
if err != nil {
sublog.Error().Err(err).Msg("Cannot create Docker client")
return []model.Image{}
}
svcs, err := cli.ServiceList(filters.NewArgs())
if err != nil {
sublog.Error().Err(err).Msg("Cannot list Swarm services")
return []model.Image{}
}
var list []model.Image
for _, svc := range svcs {
image, err := provider.ValidateContainerImage(svc.Spec.TaskTemplate.ContainerSpec.Image, svc.Spec.Labels, elt.WatchByDefault)
if err != nil {
sublog.Error().Err(err).Msgf("Cannot get image from service %s", svc.ID)
continue
} else if reflect.DeepEqual(image, model.Image{}) {
sublog.Debug().Msgf("Watch disabled for service %s", svc.ID)
continue
}
list = append(list, image)
}
return list
}

41
internal/provider/swarm/swarm.go Normal file
View file

@ -0,0 +1,41 @@
package swarm
import (
"github.com/crazy-max/diun/internal/model"
"github.com/crazy-max/diun/internal/provider"
"github.com/rs/zerolog/log"
)
// Client represents an active swarm provider object
type Client struct {
*provider.Client
elts []model.PrdSwarm
}
// New creates new swarm provider instance
func New(elts []model.PrdSwarm) *provider.Client {
return &provider.Client{Handler: &Client{
elts: elts,
}}
}
// ListJob returns job list to process
func (c *Client) ListJob() []model.Job {
if len(c.elts) == 0 {
return []model.Job{}
}
log.Info().Msgf("Found %d swarm provider(s) to analyze...", len(c.elts))
var list []model.Job
for _, elt := range c.elts {
for _, img := range c.listServiceImage(elt) {
list = append(list, model.Job{
Provider: "swarm",
ID: elt.ID,
Image: img,
})
}
}
return list
}

7
pkg/docker/container.go
View file

@ -1,16 +1,15 @@
package docker
import (
"context"
"sort"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
)
// Containers return containers based on filters
func (c *Client) Containers(filterArgs filters.Args) ([]types.Container, error) {
containers, err := c.Api.ContainerList(context.Background(), types.ContainerListOptions{
// ContainerList returns Docker containers
func (c *Client) ContainerList(filterArgs filters.Args) ([]types.Container, error) {
containers, err := c.Api.ContainerList(c.ctx, types.ContainerListOptions{
Filters: filterArgs,
})
if err != nil {

25
pkg/docker/service.go Normal file
View file

@ -0,0 +1,25 @@
package docker
import (
"sort"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/api/types/swarm"
)
// ServiceList returns Swarm services
func (c *Client) ServiceList(filterArgs filters.Args) ([]swarm.Service, error) {
services, err := c.Api.ServiceList(c.ctx, types.ServiceListOptions{
Filters: filterArgs,
})
if err != nil {
return nil, err
}
sort.Slice(services, func(i, j int) bool {
return services[i].Spec.Name < services[j].Spec.Name
})
return services, nil
}