api

package
v0.0.0-...-f628d08 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Feb 25, 2022 License: Apache-2.0 Imports: 2 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source
var DefaultTimeout = 30.0

Functions

func ImageIsLocalFile

func ImageIsLocalFile(image string) bool

ImageIsLocalFile checks if an image string is a local file or not

Types

type CloudCluster

type CloudCluster struct {
	Tag string `json:"tag,omitempty" yaml:"tag"`
	// Name         string     `json:"name,omitempty" yaml:"name"`
	// InstanceType string     `json:"instance_type,omitempty" yaml:"instance_type"`
	// Region       string     `json:"region,omitempty" yaml:"region"`
	Instances int `json:"instances,omitempty" yaml:"instances"`
	// They match the InputMounts in job spec
	Inputs []DataSpec `json:"inputs,omitempty" yaml:"inputs"`
	// It matches the OutputMount in job spec
	Output DataSpec    `json:"output,omitempty" yaml:"output"`
	Temp   TempStorage `json:"temp,omitempty" yaml:"temp"`
}

Job

type CloudStorage

type CloudStorage struct {
	Type       string `json:"type,omitempty" yaml:"type"`
	Entrypoint string `json:"entrypoint,omitempty" yaml:"entry_point"`
	Key        string `json:"key,omitempty" yaml:"key"`
	Secret     string `json:"secret,omitempty" yaml:"secret"`
}

CloudStorage specifies a cloud storage (WIP)

type Cost

type Cost struct {
	HourRate       int32  `json:"hour_rate,omitempty" yaml:"hour_rate"`
	RateUnit       string `json:"rate_unit,omitempty" yaml:"rate_unit"`
	ComputeCost    int32  `json:"compute_cost,omitempty" yaml:"compute_cost"`
	EgressGB       int32  `json:"egress_gb,omitempty" yaml:"egress_gb"`
	EgressCost     int32  `json:"egress_cost,omitempty" yaml:"egress_cost"`
	AdjustCost     int32  `json:"adjust_cost,omitempty" yaml:"adjust_cost"`
	ReservedCredit int32  `json:"reserved_credit,omitempty" yaml:"reserved_credit"`
}

type DataSpec

type DataSpec struct {
	// must be a directory
	LocalDir string `json:"local_dir,omitempty" yaml:"local_dir"` // local path on the client's machine
	// cloud storage for staging data b/w client and cloud instance
	// user can directly specify from her own cloud storage or just specify the local path.
	Cloud CloudStorage `json:"cloud,omitempty" yaml:"cloud"`
}

DataSpec defines the input source or output destination. Only one of the field needs to be specified. If the user just uses local directory, a cloud Stage will be generated by cloudor service in JobRunInfo.InputStage

type Env

type Env struct {
	Name  string `json:"name,omitempty" yaml:"name"`
	Value string `json:"value,omitempty" yaml:"value"`
}

////////////////////////////////////////////////

type Job

type Job struct {
	// must be job
	Kind    string  `json:"kind,omitempty" yaml:"kind"`
	Version string  `json:"version,omitempty" yaml:"version"`
	Spec    RunSpec `json:"spec,omitempty" yaml:"spec"`

	Vendors []CloudCluster `json:"vendors,omitempty" yaml:"vendors"`
}

func DefaultJob

func DefaultJob() *Job

func (*Job) FindRunningVendorIndexByTag

func (job *Job) FindRunningVendorIndexByTag(tag string) int32

//////////////////////////////////////////////// Find the chosen vendor by tag, if no tag is set, choose the first one if no vendor exists, return -1

func (*Job) HasLocals

func (job *Job) HasLocals(tag string) (bool, bool)

HasLocals returns a pair of boolean that indicate if the chosen vendor run has any local directory involved. They affect the behavior of job run: If both false, the job run can immediately return If Input is true, the job run needs to wait for the instances to boot to copy the local dir If output is true, the job run needs to wait for the run to finish to copy back to local dir

type JobRunInfo

type JobRunInfo struct {
	// unique id, read-only
	// job name, can be auto-generated
	JobName      string         `json:"job_name,omitempty" yaml:"job_name"`
	TimeoutInMin float64        `json:"timeout_in_min,omitempty" yaml:"timeout_in_min"`
	Duration     int64          `json:"duration,omitempty" yaml:"duration"`
	Instances    int            `json:"instances,omitempty" yaml:"instances"`
	Cost         Cost           `json:"cost,omitempty" yaml:"cost"`
	VendorIndex  *int32         `json:"vendor_index,omitempty" yaml:"vendor_index"`
	VendorInfo   VendorInfo     `json:"vendor_info,omitempty" yaml:"vendor_info"`
	InputStages  []StageStorage `json:"input_stages,omitempty" yaml:"input_stages"`
	OutputStage  []StageStorage `json:"output_stage,omitempty" yaml:"output_stage"`
	ImageStage   StageStorage   `json:"image_stage,omitempty" yaml:"image_stage"`
	WorkingDir   string         `json:"working_dir,omitempty" yaml:"working_dir"`
	Stages       []Status       `json:"stages,omitempty" yaml:"stages"`
	// Internal usage
	UpdateNotice Notice            `json:"update_notice,omitempty" yaml:"update_notice"`
	Reserved     map[string]string `json:"reserved,omitempty" yaml:"reserved"`
}

type JobStat

type JobStat struct {
	Booting         int32
	Running         int32
	AssignInstances int
}

type JobStatus

type JobStatus struct {
	UserName    string `json:"user_name,omitempty" yaml:"user_name"`
	ID          string `json:"id,omitempty" yaml:"id"`
	IID         string `json:"iid,omitempty" yaml:"iid"`
	Status      string `json:"status,omitempty" yaml:"status"`
	StatusCode  int32  `json:"status_code,omitempty" yaml:"status_code"`
	Vendor      string `json:"vendor,omitempty" yaml:"vendor"`
	Description string `json:"description,omitempty" yaml:"description"`
}

JobStatus is used for streaming job status to the client

type ListJobRequest

type ListJobRequest struct {
	UserName string `json:"user_name,omitempty"`
	ID       string `json:"id,omitempty"`
	Status   string `json:"status,omitempty"`
}

type Notice

type Notice struct {
	Status string `json:"status,omitempty" yaml:"status"`
	URI    string `json:"uri,omitempty" yaml:"uri"`
	Body   string `json:"body,omitempty" yaml:"body"`
}

type Port

type Port struct {
	// TCP for now
	Type          string `json:"type,omitempty" yaml:"type"`
	ContainerPort int32  `json:"container_port,omitempty" yaml:"container_port"`
	ExposePort    int32  `  json:"expose_port,omitempty" yaml:"expose_port"`
}

type PresignHTTPRequest

type PresignHTTPRequest struct {
	URL    string
	Method string
	Header http.Header
}

type PresignPair

type PresignPair struct {
	// 	Region string             `json:"region,omitempty" yaml:"region"`
	//	Key    string             `json:"key,omitempty" yaml:"key"`
	Put PresignHTTPRequest `json:"put,omitempty" yaml:"put"`
	Get PresignHTTPRequest `json:"get,omitempty" yaml:"get"`
}

bi-directional to connect both sides (client and container)

type RunJobMessage

type RunJobMessage struct {
	UserName string     `json:"user_name,omitempty" yaml:"user_name"`
	Created  int64      `json:"created,omitempty" yaml:"created"`
	ID       string     `json:"id,omitempty" yaml:"id"`
	RunInfo  JobRunInfo `json:"run_info,omitempty" yaml:"run_info"`
	Job      Job        `json:"job,omitempty" yaml:"job"`
}

RunJobMessage is the structured data structure for communication

type RunJobRequest

type RunJobRequest struct {
	UserName     string  `json:"user_name,omitempty" yaml:"user_name"`
	JobName      string  `json:"job_name,omitempty" yaml:"job_name"`
	RunTag       string  `json:"run_tag,omitempty" yaml:"run_tag"`
	DryRun       bool    `json:"dry_run,omitempty" yaml:"dry_run"`
	NumInstances int     `json:"num_instances,omitempty" yaml:"num_instances"`
	TimeoutInMin float64 `json:"timeout_in_min,omitempty" yaml:"timeout_in_min"`
	YAML         string  `json:"yaml,omitempty" yaml:"yaml"`
}

RunJobRequest defines the request for running a job

type RunSpec

type RunSpec struct {
	Type  string `json:"type,omitempty"`
	Image string `json:"image,omitempty" yaml:"image"`
	// for private docker registry
	ImagePullSecret string   `json:"image_pull_secret,omitempty" yaml:"image_pull_secret"`
	Envs            []Env    `json:"envs,omitempty" yaml:"envs"`
	Command         string   `json:"command,omitempty" yaml:"command"`
	Args            []string `json:"args,omitempty" yaml:"args"`
	Ports           []Port   `json:"ports,omitempty" yaml:"ports"`
	// Input mounting paths in the container. There can be more than one sources
	// The runtime will mount input directories to them
	InputMounts []string `json:"input_mounts,omitempty" yaml:"input_mounts"`
	// Output mounting path in the container.
	OutputMount   string   `json:"output_mount,omitempty" yaml:"output_mount"`
	Notifications []Notice `json:"notifications,omitempty" yaml:"notifications"`
}

type S3Presign

type S3Presign struct {
	Bucket string `json:"bucket,omitempty" yaml:"bucket"`
	URL    string `json:"url,omitempty" yaml:"url"`
	// Put or Get
	Method string              `json:"method,omitempty" yaml:"method"`
	Header map[string][]string `json:"header,omitempty" yaml:"header"`
	Expiry int64               `json:"expiry,omitempty" yaml:"expiry"` // set when using cloudor's default stage storage
}

S3Presign defines a S3 bucket presign url

type SchedulerStatus

type SchedulerStatus struct {
	Waiting int
	Status  map[string]State
}

type StageStorage

type StageStorage struct {
	// Type   string       `json:"type,omitempty" yaml:"type"`
	Cloud CloudStorage `json:"cloud,omitempty" yaml:"cloud"`
	Pair  PresignPair  `json:"s3_pair,omitempty" yaml:"s3_pair"`
}

type State

type State struct {
	Idling      int
	Busying     int
	Reserved    int
	Terminating int
	Booting     int
	Max         int
	Tag         string
	JobState    *JobStat
}

type Status

type Status struct {
	ReturnCode  int32  `json:"return_code,omitempty" yaml:"return_code"`
	Status      string `json:"status,omitempty" yaml:"status"`
	Description string `json:"description,omitempty" yaml:"description"`
	StdOut      string `json:"std_out,omitempty" yaml:"stdout"`
	UnixTime    int64  `json:"unix_time,omitempty" yaml:"unix_time"`
}

type TempStorage

type TempStorage struct {
	SizeInGB int32  `json:"size_in_gb,omitempty" yaml:"size_in_gb"`
	Type     string `json:"type,omitempty" yaml:"type"`
	Mount    string `json:"mount,omitempty" yaml:"mount"`
}

type VendorInfo

type VendorInfo struct {
	Name         string `json:"name,omitempty" yaml:"name"`
	InstanceType string `json:"instance_type,omitempty" yaml:"instance_type"`
	Region       string `json:"region,omitempty" yaml:"region"`
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL