belaur

package module
v0.0.0-...-d892afd Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 30, 2022 License: MIT Imports: 5 Imported by: 0

README

Bhojpur Belaur - Pipeline Management

The Bhojpur Belaur is a high performance DevOps pipeline management engine applied within the Bhojpur.NET Platform ecosystem for delivery of distributed applications or services. Initially, it was designed for software builds, but it could be customised for AI/ML models or to process any kind of data (e.g., image, audio/video, simulation models, CRM/ERP).

Getting Started

Firstly, the Bhojpur Belaur leverages a standard Kubernetes instance to manage multiple build pipelines for your software applications.

Build Source Code

Firstly, clone this Git reporitory in a folder. You would require Go programming language runtime to build the Belaur CLI. You need Node.js also for building the Belaur Dashboard. Now, issue the following commands in a terminal window.

go mod tidy
make compile_frontend
make static_assets
make compile_backend

if successful, then you could run the following command in another terminal window.

make dev

or

belaur -home-path=${PWD}/tmp -dev=true

in a development mode. Now, open the http://localhost:8080 URL in a web broswer.

Installation

The installation of Bhojpur Belaur is simple and often takes a few minutes only.

Using Docker

The following command starts Bhojpur Belaur as a daemon process and mounts all data to the current folder. Afterwards, Bhojpur Belaur will be available on the host system on port 8080. Use the standard user admin and password admin as initial login. It is recommended to change the password afterwards.

docker run -d -p 8080:8080 -v $PWD:/data bhojpur/belaur:latest

It uses the Docker image with the latest tag, which includes all required libraries and compilers for all supported languages. If you prefer a smaller image suited for a preferred language, then you have a look at the available docker image tags_.

Manually

It is possible to install Bhojpur Belaur directly on the host system. This can be achieved by downloading the binary from the releases page_.

The Bhojpur Belaur will automatically detect the folder of the binary and will place all data next to it. You can change the data directory with the startup parameter -home-path, if you want.

Using Helm

If you haven't got an ingress controller pod yet, make sure that you have kube-dns or coredns enabled, run this command to set it up.

make kube-ingress

To init helm:

helm init

To deploy Bhojpur Belaur:

make deploy-kube
Example Pipelines
Go
    package main

    import (
        "log"

        sdk "github.com/bhojpur/gosdk"
    )

    // This is one job. Add more if you want.
    func DoSomethingAwesome(args sdk.Arguments) error {
        log.Println("This output will be streamed back to Bhojpur Belaur and will be displayed in the pipeline logs.")

        // An error occurred? Return it back so Bhojpur Belaur knows that this job failed.
        return nil
    }

    func main() {
        jobs := sdk.Jobs{
            sdk.Job{
                Handler:     DoSomethingAwesome,
	            Title:       "DoSomethingAwesome",
		        Description: "This job does something awesome.",
	        },
	    }

	    // Serve
	    if err := sdk.Serve(jobs); err != nil {
	        panic(err)
	    }
    }
Python
    from belaursdk import sdk
    import logging

    def MyAwesomeJob(args):
        logging.info("This output will be streamed back to Bhojpur Belaur and will be displayed in the pipeline logs.")
        # Just raise an exception to tell Bhojpur Belaur if a job failed.
        # raise Exception("Oh no, this job failed!")

    def main():
        logging.basicConfig(level=logging.INFO)
        myjob = sdk.Job("MyAwesomeJob", "Do something awesome", MyAwesomeJob)
        sdk.serve([myjob])
Java
    package net.bhojpur.belaur;

    import net.bhojpur.belaur.sdk.*;
    import java.util.ArrayList;
    import java.util.Arrays;
    import java.util.logging.Logger;

    public class Pipeline
    {
        private static final Logger LOGGER = Logger.getLogger(Pipeline.class.getName());

        private static Handler MyAwesomeJob = (belaurArgs) -> {
            LOGGER.info("This output will be streamed back to Bhojpur Belaur and will be displayed in the pipeline logs.");
	    // Just raise an exception to tell Bhojpur Belaur if a job failed.
            // throw new IllegalArgumentException("Oh no, this job failed!");
        };

        public static void main( String[] args )
        {
            PipelineJob myjob = new PipelineJob();
            myjob.setTitle("MyAwesomeJob");
            myjob.setDescription("Do something awesome.");
            myjob.setHandler(MyAwesomeJob);

            Javasdk sdk = new Javasdk();
            try {
                sdk.Serve(new ArrayList<>(Arrays.asList(myjob)));
            } catch (Exception ex) {
                ex.printStackTrace();
            }
        }
    }
C++
   #include "cppsdk/sdk.h"
   #include <list>
   #include <iostream>

   void DoSomethingAwesome(std::list<belaur::argument> args) throw(std::string) {
      std::cerr << "This output will be streamed back to Bhojpur Belaur and will be displayed in the pipeline logs." << std::endl;

      // An error occurred? Return it back so Bhojpur Belaur knows that this job failed.
      // throw "Uhh something badly happened!"
   }

   int main() {
      std::list<belaur::job> jobs;
      belaur::job awesomejob;
      awesomejob.handler = &DoSomethingAwesome;
      awesomejob.title = "DoSomethingAwesome";
      awesomejob.description = "This job does something awesome.";
      jobs.push_back(awesomejob);

      try {
         belaur::Serve(jobs);
      } catch (string e) {
         std::cerr << "Error: " << e << std::endl;
      }
   }
Ruby
   require 'rubysdk'

   class Main
       AwesomeJob = lambda do |args|
           STDERR.puts "This output will be streamed back to Bhojpur Belaur and will be displayed in the pipeline logs."

           # An error occurred? Raise an exception and Bhojpur Belaur will fail the pipeline.
           # raise "Oh gosh! Something went wrong!"
       end

       def self.main
           awesomejob = Interface::Job.new(title: "Awesome Job",
                                           handler: AwesomeJob,
                                           desc: "This job does something awesome.")

           begin
               RubySDK.Serve([awesomejob])
           rescue => e
               puts "Error occured: #{e}"
               exit(false)
           end
       end
   end
Node.js
   const nodesdk = require('@bhojpur/belaursdk');

   function DoSomethingAwesome(args) {
       console.error('This output will be streamed back to Bhojpur Belaur and will be displayed in the pipeline logs.');

       // An error occurred? Throw it back so Bhojpur Belaur knows that this job failed.
       // throw new Error('My error message');
   }

   // Serve
   try {
       nodesdk.Serve([{
           handler: DoSomethingAwesome,
           title: 'DoSomethingAwesome',
           description: 'This job does something awesome.'
       }]);
   } catch (err) {
       console.error(err);
   }

Pipelines are defined by jobs and a function usually represents a job. You can define as many jobs in your pipeline as you want.

Every function accepts arguments. Those arguments can be requested from the itself and the values are passed back in from the UI.

Some pipeline jobs need a specific order of execution. DependsOn allows you to declare dependencies for every job.

Documentation

Index

Constants

View Source 
const (
	// PTypeUnknown unknown plugin type
	PTypeUnknown PipelineType = "unknown"

	// PTypeGolang golang plugin type
	PTypeGolang PipelineType = "golang"

	// PTypeJava java plugin type
	PTypeJava PipelineType = "java"

	// PTypePython python plugin type
	PTypePython PipelineType = "python"

	// PTypeCpp C++ plugin type
	PTypeCpp PipelineType = "cpp"

	// PTypeRuby ruby plugin type
	PTypeRuby PipelineType = "ruby"

	// PTypeNodeJS NodeJS plugin type
	PTypeNodeJS PipelineType = "nodejs"

	// CreatePipelineFailed status
	CreatePipelineFailed CreatePipelineType = "failed"

	// CreatePipelineRunning status
	CreatePipelineRunning CreatePipelineType = "running"

	// CreatePipelineSuccess status
	CreatePipelineSuccess CreatePipelineType = "success"

	// RunNotScheduled status
	RunNotScheduled PipelineRunStatus = "not scheduled"

	// RunScheduled status
	RunScheduled PipelineRunStatus = "scheduled"

	// RunFailed status
	RunFailed PipelineRunStatus = "failed"

	// RunSuccess status
	RunSuccess PipelineRunStatus = "success"

	// RunRunning status
	RunRunning PipelineRunStatus = "running"

	// RunCancelled status
	RunCancelled PipelineRunStatus = "cancelled"

	// RunReschedule status
	RunReschedule PipelineRunStatus = "reschedule"

	// JobWaitingExec status
	JobWaitingExec JobStatus = "waiting for execution"

	// JobSuccess status
	JobSuccess JobStatus = "success"

	// JobFailed status
	JobFailed JobStatus = "failed"

	// JobRunning status
	JobRunning JobStatus = "running"

	// ModeServer mode
	ModeServer Mode = "server"

	// ModeWorker mode
	ModeWorker Mode = "worker"

	// WorkerActive status
	WorkerActive WorkerStatus = "active"

	// WorkerInactive status
	WorkerInactive WorkerStatus = "inactive"

	// WorkerSuspended status
	WorkerSuspended WorkerStatus = "suspended"

	// LogsFolderName represents the Name of the logs folder in pipeline run folder
	LogsFolderName = "logs"

	// LogsFileName represents the file name of the logs output
	LogsFileName = "output.log"

	// APIVersion represents the current API version
	APIVersion = "v1"

	// SrcFolder folder name where the sources are stored
	SrcFolder = "src"

	// TmpFolder is the temp folder for temporary files
	TmpFolder = "tmp"

	// TmpPythonFolder is the name of the python temporary folder
	TmpPythonFolder = "python"

	// TmpGoFolder is the name of the golang temporary folder
	TmpGoFolder = "golang"

	// TmpCppFolder is the name of the c++ temporary folder
	TmpCppFolder = "cpp"

	// TmpRubyFolder is the name of the ruby temporary folder
	TmpRubyFolder = "ruby"

	// TmpNodeJSFolder is the name of the nodejs temporary folder
	TmpNodeJSFolder = "nodejs"

	// TmpJavaFolder is the name of the java temporary folder
	TmpJavaFolder = "java"

	// WorkerRegisterKey is the used key for worker registration secret
	WorkerRegisterKey = "WORKER_REGISTER_KEY"

	// ExecutablePermission is the permission used for Bhojpur Belaur created executables.
	ExecutablePermission = 0700

	// StartReasonRemote label for pipelines which were triggered through a remote token.
	StartReasonRemote = "remote"

	// StartReasonManual label for pipelines which were triggered through the admin site.
	StartReasonManual = "manual"

	// StartReasonScheduled label for pipelines which were triggered automated process, i.e. cron job.
	StartReasonScheduled = "scheduled"

	// SecretNamePrefix defines the prefix for github secrets for pipelines.
	SecretNamePrefix = "GITHUB_WEBHOOK_SECRET_"

	// LegacySecretName is the old name for a secret that has been created by previous versions.
	// Deprecated
	LegacySecretName = "GITHUB_WEBHOOK_SECRET"
)
View Source 
const JwtExpiry = 12 * 60 * 60

JwtExpiry is the default JWT expiry.

Variables

View Source 
var Cfg = &Config{}

Cfg represents the global config instance

Functions

This section is empty.

Types

type Argument 

type Argument struct {
	Description string `json:"desc,omitempty"`
	Type        string `json:"type,omitempty"`
	Key         string `json:"key,omitempty"`
	Value       string `json:"value,omitempty"`
}

Argument represents a single argument of a job

type Config 

type Config struct {
	DevMode                 bool
	ModeRaw                 string
	Mode                    Mode
	VersionSwitch           bool
	Poll                    bool
	PVal                    int
	ListenPort              string
	HomePath                string
	Hostname                string
	VaultPath               string
	DataPath                string
	PipelinePath            string
	WorkspacePath           string
	Worker                  int
	JwtPrivateKeyPath       string
	JWTKey                  interface{}
	Logger                  hclog.Logger
	CAPath                  string
	WorkerServerPort        string
	PreventPrimaryWork      bool
	AutoDockerMode          bool
	DockerHostURL           string
	DockerRunImage          string
	DockerWorkerHostURL     string
	DockerWorkerGRPCHostURL string
	RBACEnabled             bool
	RBACDebug               bool

	// Worker
	WorkerName        string
	WorkerHostURL     string
	WorkerGRPCHostURL string
	WorkerSecret      string
	WorkerTags        string

	Bolt struct {
		Mode os.FileMode
	}
}

Config holds all config options

type CreatePipeline 

type CreatePipeline struct {
	ID          string             `json:"id,omitempty"`
	Pipeline    Pipeline           `json:"pipeline,omitempty"`
	Status      int                `json:"status,omitempty"`
	StatusType  CreatePipelineType `json:"statustype,omitempty"`
	Output      string             `json:"output,omitempty"`
	Created     time.Time          `json:"created,omitempty"`
	GitHubToken string             `json:"githubtoken,omitempty"`
}

CreatePipeline represents a pipeline which is not yet compiled.

type CreatePipelineType 

type CreatePipelineType string

CreatePipelineType represents the different status types a create pipeline can have.

type GitRepo 

type GitRepo struct {
	URL            string     `json:"url,omitempty"`
	Username       string     `json:"user,omitempty"`
	Password       string     `json:"password,omitempty"`
	PrivateKey     PrivateKey `json:"privatekey,omitempty"`
	SelectedBranch string     `json:"selectedbranch,omitempty"`
	Branches       []string   `json:"branches,omitempty"`
	LocalDest      string     `json:"-"`
}

GitRepo represents a single git repository

type Job 

type Job struct {
	ID           uint32      `json:"id,omitempty"`
	Title        string      `json:"title,omitempty"`
	Description  string      `json:"desc,omitempty"`
	DependsOn    []*Job      `json:"dependson,omitempty"`
	Status       JobStatus   `json:"status,omitempty"`
	Args         []*Argument `json:"args,omitempty"`
	FailPipeline bool        `json:"failpipeline,omitempty"`
}

Job represents a single job of a pipeline

type JobStatus 

type JobStatus string

JobStatus represents the different status a job can have

type JwtCustomClaims 

type JwtCustomClaims struct {
	Username string   `json:"username"`
	Roles    []string `json:"roles"`
	jwt.StandardClaims
}

JwtCustomClaims is the custom JWT claims for Bhojpur Belaur session.

type Mode 

type Mode string

Mode represents the different modes for Bhojpur Belaur

type Pipeline 

type Pipeline struct {
	ID                int          `json:"id,omitempty"`
	Name              string       `json:"name,omitempty"`
	Repo              *GitRepo     `json:"repo,omitempty"`
	Type              PipelineType `json:"type,omitempty"`
	ExecPath          string       `json:"execpath,omitempty"`
	SHA256Sum         []byte       `json:"sha256sum,omitempty"`
	Jobs              []*Job       `json:"jobs,omitempty"`
	Created           time.Time    `json:"created,omitempty"`
	UUID              string       `json:"uuid,omitempty"`
	IsNotValid        bool         `json:"notvalid,omitempty"`
	PeriodicSchedules []string     `json:"periodicschedules,omitempty"`
	TriggerToken      string       `json:"trigger_token,omitempty"`
	Tags              []string     `json:"tags,omitempty"`
	Docker            bool         `json:"docker"`
	CronInst          *cron.Cron   `json:"-"`
}

Pipeline represents a single pipeline

type PipelineRun 

type PipelineRun struct {
	UniqueID       string            `json:"uniqueid"`
	ID             int               `json:"id"`
	PipelineID     int               `json:"pipelineid"`
	StartDate      time.Time         `json:"startdate,omitempty"`
	StartReason    string            `json:"started_reason"`
	FinishDate     time.Time         `json:"finishdate,omitempty"`
	ScheduleDate   time.Time         `json:"scheduledate,omitempty"`
	Status         PipelineRunStatus `json:"status,omitempty"`
	Jobs           []*Job            `json:"jobs,omitempty"`
	PipelineType   PipelineType      `json:"pipelinetype,omitempty"`
	PipelineTags   []string          `json:"pipelinetags,omitempty"`
	Docker         bool              `json:"docker,omitempty"`
	DockerWorkerID string            `json:"dockerworkerid,omitempty"`
}

PipelineRun represents a single run of a pipeline.

type PipelineRunStatus 

type PipelineRunStatus string

PipelineRunStatus represents the different status a run can have.

type PipelineType 

type PipelineType string

PipelineType represents supported plugin types

func (PipelineType) String 

func (p PipelineType) String() string

String returns a pipeline type string back

type PrivateKey 

type PrivateKey struct {
	Key      string `json:"key,omitempty"`
	Username string `json:"username,omitempty"`
	Password string `json:"password,omitempty"`
}

PrivateKey represents a pem encoded private key

type SHAPair 

type SHAPair struct {
	Original   []byte `json:"original"`
	Worker     []byte `json:"worker"`
	PipelineID int    `json:"pipelineid"`
}

SHAPair struct contains the original sha of a pipeline executable and the new sha which was created when the worker had to rebuild it.

type StoreConfig 

type StoreConfig struct {
	ID          int
	Poll        bool
	RBACEnabled bool
}

StoreConfig defines config settings to be stored in DB.

type User 

type User struct {
	Username     string    `json:"username,omitempty"`
	Password     string    `json:"password,omitempty"`
	DisplayName  string    `json:"display_name,omitempty"`
	Tokenstring  string    `json:"tokenstring,omitempty"`
	JwtExpiry    int64     `json:"jwtexpiry,omitempty"`
	LastLogin    time.Time `json:"lastlogin,omitempty"`
	TriggerToken string    `json:"trigger_token,omitempty"`
}

User is the user object

type UserPermission 

type UserPermission struct {
	Username string   `json:"username"`
	Roles    []string `json:"roles"`
	Groups   []string `json:"groups"`
}

UserPermission is stored in its own data structure away from the core user. It represents all permission data for a single user.

type UserRole 

type UserRole struct {
	Name        string              `json:"name"`
	Description string              `json:"description"`
	APIEndpoint []*UserRoleEndpoint `json:"api_endpoints"`
}

UserRole represents a single permission role.

type UserRoleCategory 

type UserRoleCategory struct {
	Name        string      `json:"name"`
	Description string      `json:"description"`
	Roles       []*UserRole `json:"roles"`
}

UserRoleCategory represents the top-level of the permission role system

type UserRoleEndpoint 

type UserRoleEndpoint struct {
	Path   string `json:"path"`
	Method string `json:"method"`
}

UserRoleEndpoint represents the path and method of the API endpoint to be secured.

type Worker 

type Worker struct {
	UniqueID     string       `json:"uniqueid"`
	Name         string       `json:"name"`
	Status       WorkerStatus `json:"status"`
	Slots        int32        `json:"slots"`
	RegisterDate time.Time    `json:"registerdate"`
	LastContact  time.Time    `json:"lastcontact"`
	FinishedRuns int64        `json:"finishedruns"`
	Tags         []string     `json:"tags"`
}

Worker represents a single registered worker.

type WorkerStatus 

type WorkerStatus string

WorkerStatus represents the different status a worker can have

Source Files

View all Source files

Directories

Path Synopsis
cmd
server
server/commands
pkg
adapter
api/v1/plugin
api/v1/worker
handlers
handlers/errors
helper/assethelper
helper/filehelper
helper/pipelinehelper
helper/rolehelper
helper/stringhelper
plugin
providers
providers/pipelines
providers/rbac
providers/user
providers/workers
security
security/rbac
services
store
store/memdb
utils/flag
version
workers/agent
workers/agent/api
workers/docker
workers/pipeline
workers/scheduler/belaurscheduler
workers/scheduler/service
workers/server

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.