Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add templating support #336

Closed
wants to merge 1 commit into from
Closed

add templating support #336

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions pkg/build/build.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ var (

type Build struct {
name string
cfg util.CorePackageTemplateConfig
cfg util.PackageTemplateConfig
kindConfigPath string
kubeConfigPath string
kubeVersion string
Expand All @@ -43,7 +43,7 @@ type Build struct {

type NewBuildOptions struct {
Name string
TemplateData util.CorePackageTemplateConfig
TemplateData util.PackageTemplateConfig
KindConfigPath string
KubeConfigPath string
KubeVersion string
Expand Down Expand Up @@ -115,7 +115,7 @@ func (b *Build) GetKubeClient(kubeConfig *rest.Config) (client.Client, error) {

func (b *Build) ReconcileCRDs(ctx context.Context, kubeClient client.Client) error {
// Ensure idpbuilder CRDs
if err := controllers.EnsureCRDs(ctx, b.scheme, kubeClient, b.cfg); err != nil {
if err := controllers.EnsureCRDs(ctx, b.scheme, kubeClient, b.cfg.Data); err != nil {
setupLog.Error(err, "Error creating idpbuilder CRDs")
return err
}
Expand Down Expand Up @@ -184,6 +184,7 @@ func (b *Build) Run(ctx context.Context, recreateCluster bool) error {
return err
}
b.cfg.SelfSignedCert = string(cert)
b.cfg.Data["SelfSignedCert"] = string(cert)

setupLog.V(1).Info("Running controllers")
if err := b.RunControllers(ctx, mgr, managerExit, dir); err != nil {
Expand Down
4 changes: 2 additions & 2 deletions pkg/build/coredns.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ const (
//go:embed templates
var templates embed.FS

func setupCoreDNS(ctx context.Context, kubeClient client.Client, scheme *runtime.Scheme, templateData util.CorePackageTemplateConfig) error {
func setupCoreDNS(ctx context.Context, kubeClient client.Client, scheme *runtime.Scheme, templateData util.PackageTemplateConfig) error {
checkCM := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "coredns-conf-default",
Expand All @@ -34,7 +34,7 @@ func setupCoreDNS(ctx context.Context, kubeClient client.Client, scheme *runtime
return nil
}

objs, err := k8s.BuildCustomizedObjects("", coreDNSTemplatePath, templates, scheme, templateData)
objs, err := k8s.BuildCustomizedObjects("", coreDNSTemplatePath, templates, scheme, templateData.Data)
if err != nil {
return fmt.Errorf("rendering embedded coredns files: %w", err)
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/build/tls.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ func createSelfSignedCertificate(sans []string) ([]byte, []byte, error) {
return certOut, privateKeyOut, nil
}

func setupSelfSignedCertificate(ctx context.Context, logger logr.Logger, kubeclient client.Client, config util.CorePackageTemplateConfig) ([]byte, error) {
func setupSelfSignedCertificate(ctx context.Context, logger logr.Logger, kubeclient client.Client, config util.PackageTemplateConfig) ([]byte, error) {
if err := k8s.EnsureNamespace(ctx, kubeclient, globals.NginxNamespace); err != nil {
return nil, err
}
Expand Down
50 changes: 38 additions & 12 deletions pkg/cmd/create/root.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"
"net/url"
"os"
"path/filepath"
"strings"

Expand All @@ -16,6 +17,7 @@ import (
"github.com/spf13/cobra"
"k8s.io/client-go/util/homedir"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/yaml"
)

var (
Expand All @@ -29,6 +31,7 @@ var (
extraPackagesDirs []string
extraPackages []string
packageCustomizationFiles []string
packageParamsFile string
noExit bool
protocol string
host string
Expand Down Expand Up @@ -63,7 +66,8 @@ func init() {
CreateCmd.Flags().StringSliceVar(&extraPackagesDirs, "package-dir", []string{}, "Paths to directories containing custom packages")
CreateCmd.Flags().MarkDeprecated("package-dir", "use --package instead")
CreateCmd.Flags().StringSliceVarP(&extraPackages, "package", "p", []string{}, "Paths to locations containing custom packages")
CreateCmd.Flags().StringSliceVarP(&packageCustomizationFiles, "package-custom-file", "c", []string{}, "Name of the package and the path to file to customize the package with. e.g. argocd:/tmp/argocd.yaml")
CreateCmd.Flags().StringSliceVarP(&packageCustomizationFiles, "package-custom-file", "c", []string{}, "Name of the package and the path to file to customize a core package with. e.g. argocd:/tmp/argocd.yaml")
CreateCmd.Flags().StringVarP(&packageParamsFile, "package-params-file", "F", "", "Path to file containing parameters for custom packages.")
// idpbuilder related flags
CreateCmd.Flags().BoolVarP(&noExit, "no-exit", "n", true, "When set, idpbuilder will not exit after all packages are synced. Useful for continuously syncing local directories.")
}
Expand Down Expand Up @@ -91,6 +95,7 @@ func create(cmd *cobra.Command, args []string) error {

var absDirPaths []string
var remotePaths []string
var customPackageParams map[string]any

// TODO: Remove this block after deprecation
if len(extraPackagesDirs) > 0 {
Expand All @@ -111,9 +116,17 @@ func create(cmd *cobra.Command, args []string) error {
remotePaths = r
}

if packageParamsFile != "" {
c, err := getPackageCustomizationFile(packageParamsFile)
if err != nil {
return fmt.Errorf("getting package customization from file %s : %w", packageParamsFile, err)
}
customPackageParams = c
}

o := make(map[string]v1alpha1.PackageCustomization)
for i := range packageCustomizationFiles {
c, pErr := getPackageCustomFile(packageCustomizationFiles[i])
c, pErr := getCorePackageCustomFile(packageCustomizationFiles[i])
if pErr != nil {
return pErr
}
Expand All @@ -132,13 +145,7 @@ func create(cmd *cobra.Command, args []string) error {
KindConfigPath: kindConfigPath,
ExtraPortsMapping: extraPortsMapping,

TemplateData: util.CorePackageTemplateConfig{
Protocol: protocol,
Host: host,
IngressHost: ingressHost,
Port: port,
UsePathRouting: pathRouting,
},
TemplateData: util.NewPackageTemplateConfig(protocol, host, ingressHost, port, pathRouting, customPackageParams),

CustomPackageDirs: absDirPaths,
CustomPackageUrls: remotePaths,
Expand Down Expand Up @@ -181,17 +188,16 @@ func validate() error {
}

for i := range packageCustomizationFiles {
_, pErr := getPackageCustomFile(packageCustomizationFiles[i])
_, pErr := getCorePackageCustomFile(packageCustomizationFiles[i])
if pErr != nil {
return pErr
}
}

_, _, err = helpers.ParsePackageStrings(extraPackagesDirs)
return err
}

func getPackageCustomFile(input string) (v1alpha1.PackageCustomization, error) {
func getCorePackageCustomFile(input string) (v1alpha1.PackageCustomization, error) {
// the format should be `<package-name>:<path-to-file>`
s := strings.Split(input, ":")
if len(s) != 2 {
Expand Down Expand Up @@ -219,3 +225,23 @@ func getPackageCustomFile(input string) (v1alpha1.PackageCustomization, error) {
FilePath: paths[0],
}, nil
}

func getPackageCustomizationFile(input string) (map[string]any, error) {
absPath, err := helpers.GetAbsPath(input, false)
if err != nil {
return nil, err
}

b, err := os.ReadFile(absPath)
if err != nil {
return nil, err
}

var conf map[string]any
err = yaml.Unmarshal(b, &conf)
if err != nil {
return nil, err
}

return conf, nil
}
6 changes: 3 additions & 3 deletions pkg/cmd/helpers/validation.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func ParsePackageStrings(pkgStrings []string) ([]string, []string, error) {
continue
}

absPath, err := getAbsPath(loc, true)
absPath, err := GetAbsPath(loc, true)
if err == nil {
local = append(local, absPath)
continue
Expand All @@ -56,7 +56,7 @@ func ParsePackageStrings(pkgStrings []string) ([]string, []string, error) {
return remote, local, nil
}

func getAbsPath(path string, isDir bool) (string, error) {
func GetAbsPath(path string, isDir bool) (string, error) {
absPath, err := filepath.Abs(path)
if err != nil {
return "", fmt.Errorf("failed to validate path %s : %w", path, err)
Expand All @@ -77,7 +77,7 @@ func getAbsPath(path string, isDir bool) (string, error) {
func GetAbsFilePaths(paths []string, isDir bool) ([]string, error) {
out := make([]string, len(paths))
for i := range paths {
absPath, err := getAbsPath(paths[i], isDir)
absPath, err := GetAbsPath(paths[i], isDir)
if err != nil {
return nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/controllers/custompackage/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ type Reconciler struct {
client.Client
Recorder record.EventRecorder
Scheme *runtime.Scheme
Config util.CorePackageTemplateConfig
Config util.PackageTemplateConfig
TempDir string
RepoMap *util.RepoMap
}
Expand Down
2 changes: 2 additions & 0 deletions pkg/controllers/custompackage/test/resources/customPackages/testDir/app1/cm.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,5 @@ metadata:
name: config
data:
test1: "one"
hostName: #{ .Host }#
customParam: #{ .custom.value }#
2 changes: 2 additions & 0 deletions pkg/controllers/custompackage/test/resources/customPackages/testDir/params.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
custom:
value: value1
14 changes: 7 additions & 7 deletions pkg/controllers/gitrepository/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,13 @@ type RepositoryReconciler struct {
client.Client
Recorder record.EventRecorder
Scheme *runtime.Scheme
Config util.CorePackageTemplateConfig
Config util.PackageTemplateConfig
GitProviderFunc gitProviderFunc
TempDir string
RepoMap *util.RepoMap
}

type gitProviderFunc func(context.Context, *v1alpha1.GitRepository, client.Client, *runtime.Scheme, util.CorePackageTemplateConfig) (gitProvider, error)
type gitProviderFunc func(context.Context, *v1alpha1.GitRepository, client.Client, *runtime.Scheme, util.PackageTemplateConfig) (gitProvider, error)

type notFoundError struct{}

Expand All @@ -70,7 +70,7 @@ func getFallbackRepositoryURL(repo *v1alpha1.GitRepository, info repoInfo) strin
return fmt.Sprintf("%s/%s.git", repo.Spec.Provider.GitURL, info.fullName)
}

func GetGitProvider(ctx context.Context, repo *v1alpha1.GitRepository, kubeClient client.Client, scheme *runtime.Scheme, tmplConfig util.CorePackageTemplateConfig) (gitProvider, error) {
func GetGitProvider(ctx context.Context, repo *v1alpha1.GitRepository, kubeClient client.Client, scheme *runtime.Scheme, tmplConfig util.PackageTemplateConfig) (gitProvider, error) {
switch repo.Spec.Provider.Name {
case v1alpha1.GitProviderGitea:
tr := &http.Transport{
Expand Down Expand Up @@ -236,7 +236,7 @@ func pushToRemote(ctx context.Context, remoteRepo *git.Repository, creds gitProv
}

// add files from local fs to target repository (gitea for now)
func reconcileLocalRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, tgtRepo repoInfo, creds gitProviderCredentials, scheme *runtime.Scheme, tmplConfig util.CorePackageTemplateConfig, tmpDir string, repoMap *util.RepoMap) error {
func reconcileLocalRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, tgtRepo repoInfo, creds gitProviderCredentials, scheme *runtime.Scheme, tmplConfig util.PackageTemplateConfig, tmpDir string, repoMap *util.RepoMap) error {
logger := log.FromContext(ctx)
tgtCloneDir := util.RepoDir(tgtRepo.cloneUrl, tmpDir)

Expand All @@ -256,7 +256,7 @@ func reconcileLocalRepoContent(ctx context.Context, repo *v1alpha1.GitRepository
return fmt.Errorf("cloning repo %s: %w", tgtRepoSpec.Url, err)
}

err = writeRepoContents(repo, tgtCloneDir, tmplConfig, scheme)
err = writeRepoContents(repo, tgtCloneDir, tmplConfig.Data, scheme)
if err != nil {
return fmt.Errorf("writing repo contents: %w", err)
}
Expand Down Expand Up @@ -287,7 +287,7 @@ func reconcileLocalRepoContent(ctx context.Context, repo *v1alpha1.GitRepository
}

// add files from another repository at specified path to target repository (gitea for now)
func reconcileRemoteRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, tgtRepo repoInfo, creds gitProviderCredentials, tmpDir string, repoMap *util.RepoMap) error {
func reconcileRemoteRepoContent(ctx context.Context, repo *v1alpha1.GitRepository, tgtRepo repoInfo, creds gitProviderCredentials, templateData any, tmpDir string, repoMap *util.RepoMap) error {
logger := log.FromContext(ctx)
srcRepo := repo.Spec.Source.RemoteRepository
cloneDir := util.RepoDir(srcRepo.Url, tmpDir)
Expand Down Expand Up @@ -321,7 +321,7 @@ func reconcileRemoteRepoContent(ctx context.Context, repo *v1alpha1.GitRepositor
return fmt.Errorf("cloning repo %s: %w", srcRepo.Url, err)
}

err = util.CopyTreeToTree(remoteWT, tgtRepoWT, fmt.Sprintf("/%s", repo.Spec.Source.Path), ".")
err = util.CopyTreeToTree(remoteWT, tgtRepoWT, fmt.Sprintf("/%s", repo.Spec.Source.Path), ".", templateData)
if err != nil {
return fmt.Errorf("copying contents, %s: %w", tgtRepo.cloneUrl, err)
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/controllers/gitrepository/controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ type testCase struct {
expect expect
}

func (t testCase) giteaProvider(ctx context.Context, repo *v1alpha1.GitRepository, kubeClient client.Client, scheme *runtime.Scheme, tmplConfig util.CorePackageTemplateConfig) (gitProvider, error) {
func (t testCase) giteaProvider(ctx context.Context, repo *v1alpha1.GitRepository, kubeClient client.Client, scheme *runtime.Scheme, tmplConfig util.PackageTemplateConfig) (gitProvider, error) {
return &giteaProvider{
Client: kubeClient,
Scheme: scheme,
Expand Down
11 changes: 5 additions & 6 deletions pkg/controllers/gitrepository/gitea.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ type giteaProvider struct {
client.Client
Scheme *runtime.Scheme
giteaClient GiteaClient
config util.CorePackageTemplateConfig
config util.PackageTemplateConfig
}

func (g *giteaProvider) createRepository(ctx context.Context, repo *v1alpha1.GitRepository) (repoInfo, error) {
Expand Down Expand Up @@ -110,14 +110,14 @@ func (g *giteaProvider) updateRepoContent(
case v1alpha1.SourceTypeLocal, v1alpha1.SourceTypeEmbedded:
return reconcileLocalRepoContent(ctx, repo, repoInfo, creds, g.Scheme, g.config, tmpDir, repoMap)
case v1alpha1.SourceTypeRemote:
return reconcileRemoteRepoContent(ctx, repo, repoInfo, creds, tmpDir, repoMap)
return reconcileRemoteRepoContent(ctx, repo, repoInfo, creds, g.config, tmpDir, repoMap)
default:
return nil
}
}

func writeRepoContents(repo *v1alpha1.GitRepository, dstPath string, config util.CorePackageTemplateConfig, scheme *runtime.Scheme) error {
if repo.Spec.Source.EmbeddedAppName != "" {
func writeRepoContents(repo *v1alpha1.GitRepository, dstPath string, config any, scheme *runtime.Scheme) error {
if repo.Spec.Source.Type == v1alpha1.SourceTypeEmbedded {
resources, err := localbuild.GetEmbeddedRawInstallResources(
repo.Spec.Source.EmbeddedAppName, config,
v1alpha1.PackageCustomization{Name: repo.Spec.Customization.Name, FilePath: repo.Spec.Customization.FilePath}, scheme)
Expand All @@ -134,8 +134,7 @@ func writeRepoContents(repo *v1alpha1.GitRepository, dstPath string, config util
}
return nil
}

err := util.CopyDirectory(repo.Spec.Source.Path, dstPath)
err := util.CopyDirectory(repo.Spec.Source.Path, dstPath, config)
if err != nil {
return fmt.Errorf("copying files: %w", err)
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/controllers/gitrepository/github.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ type gitHubProvider struct {
client.Client
Scheme *runtime.Scheme
gitHubClient gitHubClient
config util.CorePackageTemplateConfig
config util.PackageTemplateConfig
}

func (g *gitHubProvider) createRepository(ctx context.Context, repo *v1alpha1.GitRepository) (repoInfo, error) {
Expand Down
4 changes: 2 additions & 2 deletions pkg/controllers/localbuild/argo_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ func TestGetRawInstallResources(t *testing.T) {
resourcePath: "resources/argo",
}
resources, err := util.ConvertFSToBytes(e.resourceFS, e.resourcePath,
util.CorePackageTemplateConfig{
util.PackageTemplateConfig{
Protocol: "",
Host: "",
Port: "",
Expand All @@ -39,7 +39,7 @@ func TestGetK8sInstallResources(t *testing.T) {
resourceFS: installArgoFS,
resourcePath: "resources/argo",
}
objs, err := e.installResources(k8s.GetScheme(), util.CorePackageTemplateConfig{
objs, err := e.installResources(k8s.GetScheme(), util.PackageTemplateConfig{
Protocol: "",
Host: "",
Port: "",
Expand Down
6 changes: 3 additions & 3 deletions pkg/controllers/localbuild/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ type LocalbuildReconciler struct {
CancelFunc context.CancelFunc
ExitOnSync bool
shouldShutdown bool
Config util.CorePackageTemplateConfig
Config util.PackageTemplateConfig
TempDir string
RepoMap *util.RepoMap
}
Expand Down Expand Up @@ -434,7 +434,7 @@ func (r *LocalbuildReconciler) reconcileCustomPkgUrl(ctx context.Context, resour

rErr := r.reconcileCustomPkg(ctx, resource, b, yamlFile, remote)
if rErr != nil {
logger.Error(rErr, "reconciling custom pkg", "file", yamlFile, "pkgUrl", pkgUrl)
logger.V(1).Info("reconciling custom pkg", "file", yamlFile, "pkgUrl", pkgUrl, "error", rErr)
}
}
return ctrl.Result{}, nil
Expand Down Expand Up @@ -463,7 +463,7 @@ func (r *LocalbuildReconciler) reconcileCustomPkgDir(ctx context.Context, resour

rErr := r.reconcileCustomPkg(ctx, resource, b, filePath, nil)
if rErr != nil {
logger.Error(rErr, "reconciling custom pkg", "file", filePath, "pkgDir", pkgDir)
logger.V(1).Info("reconciling custom pkg", "file", filePath, "pkgDir", pkgDir, "error", rErr)
}
}

Expand Down
Loading
Loading