GitHub-Mirror/go-yocto
2
0
Fork 0
mirror of https://github.com/Fishwaldo/go-yocto.git synced 2025-03-15 11:21:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

more updates

Browse source
This commit is contained in:
Justin Hammond 2023-05-14 12:14:22 +08:00
parent b6625ca58f
commit af531dbfeb
13 changed files with 384 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

80
backends/appstream/appstream.go Normal file
View file

@ -0,0 +1,80 @@
package appstream
import (
"github.com/Fishwaldo/go-yocto/source"
"github.com/Fishwaldo/go-yocto/utils"
)
type AsSummary struct {
Lang string `xml:"lang,attr"`
Summary string `xml:",chardata"`
}
type AsDescription struct {
Lang string `xml:"lang,attr"`
Description string `xml:",chardata"`
}
type AsReleases struct {
Version string `xml:"version,attr"`
Date string `xml:"date,attr"`
}
type AppStream struct {
Component xml.Name `xml:"component"`
Name string `xml:"name"`
Summary []AsSummary `xml:"summary"`
Description []AsDescription `xml:"description>p"`
Releases []AsReleases `xml:"releases>release"`
}
type ASProject struct {
Backend string
BackendID string
Project AppStream
}
type AppStreamBE struct {
ready bool
projects map[string]ASProject
}
func NewBackend() *AppStreamBE {
return &AppStreamBE{}
}
func (k *AppStreamBE) GetName() string {
return "AppStream"
}
func (k *AppStreamBE) Init() error {
utils.Logger.Trace("Initializing AppStream Backend")
k.ready = true
return nil
}
func (k *AppStreamBE) LoadCache() error {
utils.Logger.Trace("Loading AppStream Cache")
return nil
}
func (k *AppStreamBE) LoadSource() error {
utils.Logger.Trace("Loading AppStream Source")
return nil
}
func (k *AppStreamBE) RefreshSource() error {
utils.Logger.Trace("Refreshing AppStream Source")
return nil
}
func (k *AppStreamBE) SearchSource(keyword string) (source []source.RecipeSource, err error) {
utils.Logger.Trace("Searching AppStream Source", utils.Logger.Args("keyword", keyword))
return nil, nil
}
func (k *AppStreamBE) Ready() bool {
return k.ready
}

61
backends/backends.go
View file

@ -2,7 +2,10 @@ package backends
import (
"github.com/Fishwaldo/go-yocto/backends/kde"
"github.com/Fishwaldo/go-yocto/backends/appstream"
"github.com/Fishwaldo/go-yocto/source"
"github.com/Fishwaldo/go-yocto/utils"
)
@ -11,7 +14,8 @@ type Backend interface {
Init() error
LoadCache() error
LoadSource() error
RefreshSource() error
SearchSource(keyword string) (source []source.RecipeSource, err error)
Ready() bool
}
var Backends map[string]Backend
@ -19,5 +23,60 @@ var Backends map[string]Backend
func init() {
Backends = make(map[string]Backend)
Backends["kde"] = kde.NewBackend()
Backends["appstream"] = appstream.NewBackend()
}
func Init() (err error) {
utils.Logger.Trace("Initializing Backends")
for _, be := range Backends {
if err := be.Init(); err != nil {
utils.Logger.Error("Failed to Initialize Backend", utils.Logger.Args("backend", be.GetName(), "error", err))
}
}
return nil
}
func LoadCache() (err error) {
utils.Logger.Trace("Loading Cache")
for _, be := range Backends {
if be.Ready() {
if err := be.LoadCache(); err != nil {
utils.Logger.Error("Failed to Load Cache", utils.Logger.Args("backend", be.GetName(), "error", err))
}
} else {
utils.Logger.Trace("LoadCache: Backend not ready", utils.Logger.Args("backend", be.GetName()))
}
}
return nil
}
func LoadSource() (err error) {
utils.Logger.Trace("Loading Source")
for _, be := range Backends {
if be.Ready() {
if err := be.LoadSource(); err != nil {
utils.Logger.Error("Failed to Load Source", utils.Logger.Args("backend", be.GetName(), "error", err))
}
} else {
utils.Logger.Trace("LoadSource: Backend not ready", utils.Logger.Args("backend", be.GetName()))
}
}
return nil
}
func SearchSource(be string, keyword string) (sources []source.RecipeSource, err error) {
utils.Logger.Trace("Searching Source")
for _, be := range Backends {
if be.Ready() {
if source, err := be.SearchSource(keyword); err != nil {
utils.Logger.Error("Failed to Search Source", utils.Logger.Args("backend", be.GetName(), "error", err))
} else {
sources = append(sources, source...)
}
} else {
utils.Logger.Trace("SearchSource: Backend not ready", utils.Logger.Args("backend", be.GetName()))
}
}
return sources, nil
}

154
backends/kde/kde.go
View file

@ -1,19 +1,23 @@
package kde
import (
"encoding/base64"
"encoding/json"
"encoding/xml"
"io/ioutil"
"os"
"path/filepath"
"strings"
"encoding/base64"
"encoding/xml"
"encoding/json"
"net/url"
"github.com/Fishwaldo/go-yocto/utils"
"github.com/Fishwaldo/go-yocto/repo"
"github.com/Fishwaldo/go-yocto/utils"
"github.com/Fishwaldo/go-yocto/source"
"golang.org/x/exp/maps"
"github.com/spf13/viper"
"github.com/pterm/pterm"
"github.com/spf13/viper"
"github.com/xanzy/go-gitlab"
"gopkg.in/yaml.v3"
)
@ -28,37 +32,13 @@ type Deps struct {
Environment interface{} `yaml:"Environment"`
}
type AsSummary struct {
Lang string `xml:"lang,attr"`
Summary string `xml:",chardata"`
}
type AsDescription struct {
Lang string `xml:"lang,attr"`
Description string `xml:",chardata"`
}
type AsReleases struct {
Version string `xml:"version,attr"`
Date string `xml:"date,attr"`
}
type AppStream struct {
Component xml.Name `xml:"component"`
Name string `xml:"name"`
Summary []AsSummary `xml:"summary"`
Description []AsDescription `xml:"description>p"`
Releases []AsReleases `xml:"releases>release"`
}
type Project struct {
Name string
source.RecipeSource `yaml:",inline"`
ProjectPath string
Repoactive bool
Repopath string
Identifier string
Hasrepo bool
Description string
Source string
Bugzilla struct {
Product string
@ -73,34 +53,50 @@ type Project struct {
}
}
type Layer struct {
type KDEBe struct {
MetaDataRepo repo.Repo
br map[string]map[string]string
pr map[string]Project
ready bool
}
func init() {
viper.SetDefault("kdeconfig.release", "@stable");
viper.SetDefault("kdeconfig.defaultbranch", "master");
viper.SetDefault("kdeconfig.kdegitlaburl", "https://invent.kde.org/api/v4")
viper.SetDefault("kdeconfig.kdegitlaburl", "https://invent.kde.org/")
}
func NewBackend() (l *Layer) {
l = &Layer{}
func NewBackend() (l *KDEBe) {
l = &KDEBe{}
return l
}
func (l *Layer) GetName() string {
func (l *KDEBe) GetName() string {
return "kde-invent"
}
func (l *Layer) Init() {
func (l *KDEBe) Init() (err error) {
utils.Logger.Trace("Initializing KDE Backend")
l.MetaDataRepo = repo.Repo{
Url: "https://invent.kde.org/sysadmin/repo-metadata",
Name: "kde-metadata",
}
l.ready = true
return nil
}
func (l *KDEBe) Ready() bool {
return l.ready
}
func (l *KDEBe) getDir() (dir string) {
dir = utils.Config.BaseDir + "/" + l.MetaDataRepo.Name
return dir
}
func (l *KDEBe) LoadSource() (err error) {
utils.Logger.Trace("Checking metadata repo", utils.Logger.Args("repo", l.MetaDataRepo, "layer", l.GetName()))
err := l.MetaDataRepo.CheckRepo()
err = l.MetaDataRepo.CheckRepo()
if (err != nil) {
utils.Logger.Info("Cloning repo", utils.Logger.Args("repo", l.MetaDataRepo, "layer", l.GetName()))
err := l.MetaDataRepo.CloneRepo()
@ -109,23 +105,21 @@ func (l *Layer) Init() {
os.Exit(-1)
}
}
err = l.ParseMetadata()
maps.Clear(l.br)
maps.Clear(l.pr)
err = l.parseMetadata()
if (err != nil) {
utils.Logger.Error("Failed to parse metadata", utils.Logger.Args("error", err))
os.Exit(-1)
}
}
func (l *Layer) GetDir() (dir string) {
dir = utils.Config.BaseDir + "/" + l.MetaDataRepo.Name
return dir
return nil
}
func (l *Layer) ParseMetadata() (err error) {
func (l *KDEBe) parseMetadata() (err error) {
l.pr = make(map[string]Project)
utils.Logger.Info("Parsing metadata", utils.Logger.Args("layer", l.Name))
brfile, err := ioutil.ReadFile(l.GetDir() + "/branch-rules.yml")
utils.Logger.Trace("Parsing metadata", utils.Logger.Args("layer", l.GetName()))
brfile, err := ioutil.ReadFile(l.getDir() + "/branch-rules.yml")
if err != nil {
utils.Logger.Error("Failed to read branch-rules.yaml", utils.Logger.Args("error", err))
os.Exit(-1)
@ -140,18 +134,18 @@ func (l *Layer) ParseMetadata() (err error) {
}
/* make sure we have a valid release */
if _, ok := l.br[utils.Config.KDEConfig.Release]; !ok {
utils.Logger.Error("Invalid release", utils.Logger.Args("release", Config.KDEConfig.Release))
utils.Logger.Error("Invalid release", utils.Logger.Args("release", utils.Config.KDEConfig.Release))
os.Exit(-1)
}
gl, err := gitlab.NewClient(utils.Config.KDEConfig.AccessToken, gitlab.WithBaseURL(utils.Config.KDEConfig.KDEGitLabURL))
gl, err := gitlab.NewClient(utils.Config.KDEConfig.AccessToken, gitlab.WithBaseURL(utils.Config.KDEConfig.KDEGitLabURL+"/api/v4"))
if err != nil {
utils.Logger.Error("Failed to create GitLab client", utils.Logger.Args("error", err))
os.Exit(-1)
}
/* now parse the directory */
files := findmetdata(l.GetDir())
files := findmetdata(l.getDir())
p, _ := pterm.DefaultProgressbar.WithTotal(len(files)).WithTitle("Parsing Metadata...").Start()
for i := 0; i < p.Total; i++ {
p.Increment()
@ -169,6 +163,8 @@ func (l *Layer) ParseMetadata() (err error) {
continue
}
data.MetaData.Branch = utils.Config.KDEConfig.DefaultBranch
data.RecipeSource.Backend = l.GetName()
data.RecipeSource.Url, _ = url.JoinPath(utils.Config.KDEConfig.KDEGitLabURL, data.Repopath)
/* find out which branch this is in... */
for project, branch := range l.br[utils.Config.KDEConfig.Release] {
ok, _ := filepath.Match(project, data.Repopath)
@ -230,25 +226,79 @@ func (l *Layer) ParseMetadata() (err error) {
utils.Logger.Error("Duplicate identifier", utils.Logger.Args("identifier", data.Identifier))
continue
}
data.Source = l.Name
data.Source = l.GetName()
l.pr[data.Identifier] = data
}
utils.Logger.Trace("Parsed metadata", utils.Logger.Args("layers", len(files)));
cache, err := json.Marshal(l.pr)
if err != nil {
utils.Logger.Error("Failed to marshal metadata", utils.Logger.Args("error", err))
os.Exit(-1)
}
err = ioutil.WriteFile(utils.Config.BaseDir + "/metadata.json", cache, 0644)
err = ioutil.WriteFile(utils.Config.BaseDir + "/" + l.GetName() + "-cache.json", cache, 0644)
if err != nil {
utils.Logger.Error("Failed to write metadata", utils.Logger.Args("error", err))
os.Exit(-1)
}
brcache, err := json.Marshal(l.br)
if err != nil {
utils.Logger.Error("Failed to marshal branch metadata", utils.Logger.Args("error", err))
os.Exit(-1)
}
err = ioutil.WriteFile(utils.Config.BaseDir + "/" + l.GetName() + "-branch-cache.json", brcache, 0644)
if err != nil {
utils.Logger.Error("Failed to write branch metadata", utils.Logger.Args("error", err))
os.Exit(-1)
}
utils.Logger.Trace("Parsed metadata", utils.Logger.Args("layers", len(l.pr), "branches", len(l.br)));
return nil
}
func (l *KDEBe) LoadCache() (err error) {
utils.Logger.Trace("Loading KDE Cache")
cache, err := ioutil.ReadFile(utils.Config.BaseDir + "/" + l.GetName() + "-cache.json")
if err != nil {
utils.Logger.Error("Failed to read cache", utils.Logger.Args("error", err))
} else {
err = json.Unmarshal(cache, &l.pr)
if err != nil {
utils.Logger.Error("Failed to unmarshal cache", utils.Logger.Args("error", err))
}
}
brcache, err := ioutil.ReadFile(utils.Config.BaseDir + "/" + l.GetName() + "-branch-cache.json")
if err != nil {
utils.Logger.Error("Failed to read branch cache", utils.Logger.Args("error", err))
} else {
err = json.Unmarshal(brcache, &l.br)
if err != nil {
utils.Logger.Error("Failed to unmarshal branch cache", utils.Logger.Args("error", err))
}
}
utils.Logger.Trace("KDE Cache Loaded", utils.Logger.Args("layers", len(l.pr), "branches", len(l.br)))
return nil
}
func (l *KDEBe) SearchSource(keywords string) (source []source.RecipeSource, err error) {
utils.Logger.Trace("Searching KDE Source", utils.Logger.Args("keyword", keywords))
p, _ := pterm.DefaultProgressbar.WithTotal(len(l.pr)).WithTitle("Searching KDE...").Start()
for _, data := range l.pr {
p.Increment()
if strings.Contains(strings.ToLower(data.Name), strings.ToLower(keywords)) {
source = append(source, data.RecipeSource)
}
if strings.Contains(strings.ToLower(data.Description), strings.ToLower(keywords)) {
source = append(source, data.RecipeSource)
}
}
return source, nil;
}
func findmetdata(path string) (files []string) {
utils.Logger.Trace("Searching...", utils.Logger.Args("path", path))
err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error {

2
cmd/cache.go
View file

@ -8,6 +8,7 @@ import (
"fmt"
"github.com/spf13/cobra"
"github.com/Fishwaldo/go-yocto/cmd/cache"
)
// cacheCmd represents the cache command
@ -22,6 +23,7 @@ var cacheCmd = &cobra.Command{
func init() {
rootCmd.AddCommand(cacheCmd)
cacheCmd.AddCommand(cmdCache.UpdateCmd)
// Here you will define your flags and configuration settings.

10
cmd/update.go → cmd/cache/update.go vendored
View file

@ -2,10 +2,9 @@
Copyright © 2023 NAME HERE <EMAIL ADDRESS>
*/
package cmd
package cmdCache
import (
"fmt"
"github.com/Fishwaldo/go-yocto/backends"
@ -13,7 +12,7 @@ import (
)
// updateCmd represents the update command
var updateCmd = &cobra.Command{
var UpdateCmd = &cobra.Command{
Use: "update",
Short: "A brief description of your command",
Long: `A longer description that spans multiple lines and likely contains examples
@ -23,13 +22,12 @@ Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
Run: func(cmd *cobra.Command, args []string) {
kdegear := backends.Layer{}
kdegear.Init()
backends.LoadSource()
},
}
func init() {
cacheCmd.AddCommand(updateCmd)
// Here you will define your flags and configuration settings.

45
cmd/root.go
View file

@ -1,32 +1,26 @@
/*
Copyright © 2023 NAME HERE <EMAIL ADDRESS>
*/
package cmd
import (
"os"
"github.com/Fishwaldo/go-yocto/backends"
"github.com/Fishwaldo/go-yocto/utils"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
// rootCmd represents the base command when called without any subcommands
var rootCmd = &cobra.Command{
Use: "go-yocto",
Short: "A brief description of your application",
Long: `A longer description that spans multiple lines and likely contains
examples and usage of using your application. For example:
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
// Uncomment the following line if your bare application
// has an action associated with it:
// Run: func(cmd *cobra.Command, args []string) { },
Short: "Manage Yocto Recipes from Sources",
Long: `Manage Yocto Recipes from Sources`,
}
var cfgFile string
// Execute adds all child commands to the root command and sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
@ -41,11 +35,34 @@ func init() {
// Cobra supports persistent flags, which, if defined here,
// will be global for your application.
// rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.go-yocto.yaml)")
rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.config/go-yocto.yaml)")
// Cobra also supports local flags, which will only run
// when this action is called directly.
rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
cobra.OnInitialize(initConfig)
}
func initConfig() {
if cfgFile != "" {
// Use config file from the flag.
viper.SetConfigFile(cfgFile)
}
viper.AutomaticEnv()
utils.InitLogger()
if err := utils.Config.InitConfig(); err != nil {
utils.Logger.Error("Failed to initialize Logger", utils.Logger.Args("error", err))
os.Exit(-1)
}
if err := backends.Init(); err != nil {
utils.Logger.Error("Failed to initialize Backends", utils.Logger.Args("error", err))
os.Exit(-1)
}
if err := backends.LoadCache(); err != nil {
utils.Logger.Error("Failed to Load Cache", utils.Logger.Args("error", err))
os.Exit(-1)
}
}

41
cmd/source.go Normal file
View file

@ -0,0 +1,41 @@
/*
Copyright © 2023 NAME HERE <EMAIL ADDRESS>
*/
package cmd
import (
"fmt"
"github.com/spf13/cobra"
"github.com/Fishwaldo/go-yocto/cmd/source"
)
// sourceCmd represents the source command
var sourceCmd = &cobra.Command{
Use: "source",
Short: "A brief description of your command",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
Run: func(cmd *cobra.Command, args []string) {
fmt.Println("source called")
},
}
func init() {
rootCmd.AddCommand(sourceCmd)
sourceCmd.AddCommand(cmdSource.SearchCmd)
// Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// sourceCmd.PersistentFlags().String("foo", "", "A help for foo")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// sourceCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}

44
cmd/source/search.go Normal file
View file

@ -0,0 +1,44 @@
/*
Copyright © 2023 NAME HERE <EMAIL ADDRESS>
*/
package cmdSource
import (
"github.com/Fishwaldo/go-yocto/backends"
"github.com/pterm/pterm"
"github.com/spf13/cobra"
)
// searchCmd represents the search command
var SearchCmd = &cobra.Command{
Use: "search",
Short: "Search For Sources accross all packages",
Long: `search for sources accross all packages`,
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
sources, err := backends.SearchSource("", args[0])
if err == nil {
td := pterm.TableData{{"Name", "Description", "Backend", "Url"}}
for _, source := range sources {
td = append(td, []string{source.Name, source.Description, source.Backend, source.Url})
}
pterm.DefaultTable.WithHasHeader().WithData(
td,
).Render()
}
},
}
func init() {
// Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// searchCmd.PersistentFlags().String("foo", "", "A help for foo")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// searchCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}

3
go.mod
View file

@ -5,8 +5,10 @@ go 1.18
require (
github.com/go-git/go-git/v5 v5.6.1
github.com/pterm/pterm v0.12.60
github.com/spf13/cobra v1.7.0
github.com/spf13/viper v1.15.0
github.com/xanzy/go-gitlab v0.83.0
golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561
gopkg.in/yaml.v3 v3.0.1
)
@ -44,7 +46,6 @@ require (
github.com/skeema/knownhosts v1.1.0 // indirect
github.com/spf13/afero v1.9.3 // indirect
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/cobra v1.7.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/subosito/gotenv v1.4.2 // indirect

1
go.sum
View file

@ -326,6 +326,7 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 h1:MDc5xs78ZrZr3HMQugiXOAkSZtfTpbJLDr/lwfgO53E=
golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

9
source/source.go Normal file
View file

@ -0,0 +1,9 @@
package source
type RecipeSource struct {
Name string
Description string
Url string
Backend string
BackendID string
}

2
utils/config.go
View file

@ -29,7 +29,7 @@ func (c *configData) InitConfig() (err error) {
Logger.Error("Failed to unmarshal config", Logger.Args("error", err))
}
if _, err = os.Stat(c.BaseDir); os.IsNotExist(err) {
Logger.Error("BaseDir does not exist", Logger.Args("error", err))
Logger.Error("BaseDir does not exist", Logger.Args("error", err, "basedir", c.BaseDir))
os.Exit(-1)
}
return err

7
utils/utils.go Normal file
View file

@ -0,0 +1,7 @@
package utils
func ClearMap[M ~map[K]V, K comparable, V any](m M) {
for k := range m {
delete(m, k)
}
}