Unverified Commit 0ae7f2d5 authored by Erik Wilson's avatar Erik Wilson Committed by GitHub
Browse files

Merge pull request #2407 from erikwilson/node-passwd-cleanup

 Use secrets for node-passwd entries
parents 989c9369 992ca52c
......@@ -152,7 +152,7 @@ func TestFailFast(t *testing.T) {
defer os.RemoveAll(tmpDir)
cfg := cmds.Agent{
ServerURL: "http://127.0.0.1:-1/",
ServerURL: "http://127.0.0.1:0/",
DataDir: tmpDir,
}
......
package hash
// Hasher is a generic interface for hashing algorithms
type Hasher interface {
// CreateHash will return a hashed version of the secretKey, or an error
CreateHash(secretKey string) (string, error)
// VerifyHash will compare a secretKey and a hash, and return nil if they match
VerifyHash(hash, secretKey string) error
}
package hash
import (
"crypto/rand"
"crypto/subtle"
"encoding/base64"
"errors"
"fmt"
"golang.org/x/crypto/scrypt"
)
// Version is the hashing format version
const Version = 1
const hashFormat = "$%d:%x:%d:%d:%d:%s"
// SCrypt contains all of the variables needed for scrypt hashing
type SCrypt struct {
N int
R int
P int
KeyLen int
SaltLen int
}
// NewSCrypt returns a scrypt hasher with recommended default values
func NewSCrypt() Hasher {
return SCrypt{
N: 15,
R: 8,
P: 1,
KeyLen: 64,
SaltLen: 8,
}
}
// CreateHash will return a hashed version of the secretKey, or an error
func (s SCrypt) CreateHash(secretKey string) (string, error) {
salt := make([]byte, s.SaltLen)
_, err := rand.Read(salt)
if err != nil {
return "", err
}
dk, err := scrypt.Key([]byte(secretKey), salt, 1<<s.N, s.R, s.P, s.KeyLen)
if err != nil {
return "", err
}
enc := base64.RawStdEncoding.EncodeToString(dk)
hash := fmt.Sprintf(hashFormat, Version, salt, s.N, s.R, s.P, enc)
return hash, nil
}
// VerifyHash will compare a secretKey and a hash, and return nil if they match
func (s SCrypt) VerifyHash(hash, secretKey string) error {
var (
version, n uint
r, p int
enc string
salt []byte
)
_, err := fmt.Sscanf(hash, hashFormat, &version, &salt, &n, &r, &p, &enc)
if err != nil {
return err
}
if version != Version {
return fmt.Errorf("hash version %d does not match package version %d", version, Version)
}
dk, err := base64.RawStdEncoding.DecodeString(enc)
if err != nil {
return err
}
verify, err := scrypt.Key([]byte(secretKey), salt, 1<<n, r, p, len(dk))
if err != nil {
return err
}
if subtle.ConstantTimeCompare(dk, verify) != 1 {
return errors.New("hash does not match")
}
return nil
}
package hash
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
var hasher = NewSCrypt()
func TestBasicHash(t *testing.T) {
secretKey := "hello world"
hash, err := hasher.CreateHash(secretKey)
assert.Nil(t, err)
assert.NotNil(t, hash)
assert.Nil(t, hasher.VerifyHash(hash, secretKey))
assert.NotNil(t, hasher.VerifyHash(hash, "goodbye"))
}
func TestLongKey(t *testing.T) {
secretKey := strings.Repeat("A", 720)
hash, err := hasher.CreateHash(secretKey)
assert.Nil(t, err)
assert.NotNil(t, hash)
assert.Nil(t, hasher.VerifyHash(hash, secretKey))
assert.NotNil(t, hasher.VerifyHash(hash, secretKey+":wrong!"))
}
......@@ -5,13 +5,21 @@ import (
"strings"
"github.com/pkg/errors"
"github.com/rancher/k3s/pkg/nodepassword"
coreclient "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
"github.com/sirupsen/logrus"
core "k8s.io/api/core/v1"
)
func Register(ctx context.Context, configMap coreclient.ConfigMapController, nodes coreclient.NodeController) error {
func Register(ctx context.Context,
modCoreDNS bool,
secretClient coreclient.SecretClient,
configMap coreclient.ConfigMapController,
nodes coreclient.NodeController,
) error {
h := &handler{
modCoreDNS: modCoreDNS,
secretClient: secretClient,
configCache: configMap.Cache(),
configClient: configMap,
}
......@@ -22,6 +30,8 @@ func Register(ctx context.Context, configMap coreclient.ConfigMapController, nod
}
type handler struct {
modCoreDNS bool
secretClient coreclient.SecretClient
configCache coreclient.ConfigMapCache
configClient coreclient.ConfigMapClient
}
......@@ -39,31 +49,45 @@ func (h *handler) onRemove(key string, node *core.Node) (*core.Node, error) {
func (h *handler) updateHosts(node *core.Node, removed bool) (*core.Node, error) {
var (
newHosts string
nodeName string
nodeAddress string
hostsMap map[string]string
)
hostsMap = make(map[string]string)
nodeName = node.Name
for _, address := range node.Status.Addresses {
if address.Type == "InternalIP" {
nodeAddress = address.Address
break
}
}
if nodeAddress == "" {
logrus.Errorf("No InternalIP found for node " + node.Name)
return nil, nil
if removed {
if err := h.removeNodePassword(nodeName); err != nil {
logrus.Warn(errors.Wrap(err, "Unable to remove node password"))
}
}
if h.modCoreDNS {
if err := h.updateCoreDNSConfigMap(nodeName, nodeAddress, removed); err != nil {
return nil, err
}
}
return nil, nil
}
func (h *handler) updateCoreDNSConfigMap(nodeName, nodeAddress string, removed bool) error {
if nodeAddress == "" && !removed {
logrus.Errorf("No InternalIP found for node " + nodeName)
return nil
}
configMapCache, err := h.configCache.Get("kube-system", "coredns")
if err != nil || configMapCache == nil {
logrus.Warn(errors.Wrap(err, "Unable to fetch coredns config map"))
return nil, nil
return nil
}
configMap := configMapCache.DeepCopy()
configMap := configMapCache.DeepCopy()
hosts := configMap.Data["NodeHosts"]
hostsMap := map[string]string{}
for _, line := range strings.Split(hosts, "\n") {
if line == "" {
continue
......@@ -75,27 +99,29 @@ func (h *handler) updateHosts(node *core.Node, removed bool) (*core.Node, error)
}
ip := fields[0]
host := fields[1]
if host == node.Name {
if host == nodeName {
if removed {
continue
}
if ip == nodeAddress {
return nil, nil
return nil
}
}
hostsMap[host] = ip
}
if !removed {
hostsMap[node.Name] = nodeAddress
hostsMap[nodeName] = nodeAddress
}
var newHosts string
for host, ip := range hostsMap {
newHosts += ip + " " + host + "\n"
}
configMap.Data["NodeHosts"] = newHosts
if _, err := h.configClient.Update(configMap); err != nil {
return nil, err
return err
}
var actionType string
......@@ -104,7 +130,10 @@ func (h *handler) updateHosts(node *core.Node, removed bool) (*core.Node, error)
} else {
actionType = "Updated"
}
logrus.Infof("%s coredns node hosts entry [%s]", actionType, nodeAddress+" "+node.Name)
logrus.Infof("%s coredns node hosts entry [%s]", actionType, nodeAddress+" "+nodeName)
return nil
}
return nil, nil
func (h *handler) removeNodePassword(nodeName string) error {
return nodepassword.Delete(h.secretClient, nodeName)
}
package nodepassword
import (
"fmt"
"os"
"strings"
"time"
"github.com/pkg/errors"
"github.com/rancher/k3s/pkg/authenticator/hash"
"github.com/rancher/k3s/pkg/passwd"
"github.com/rancher/k3s/pkg/version"
coreclient "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
"github.com/sirupsen/logrus"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
var (
// hasher provides the algorithm for generating and verifying hashes
hasher = hash.NewSCrypt()
)
func getSecretName(nodeName string) string {
return strings.ToLower(nodeName + ".node-password." + version.Program)
}
func verifyHash(secretClient coreclient.SecretClient, nodeName, pass string) error {
name := getSecretName(nodeName)
secret, err := secretClient.Get(metav1.NamespaceSystem, name, metav1.GetOptions{})
if err != nil {
return err
}
if hash, ok := secret.Data["hash"]; ok {
if err := hasher.VerifyHash(string(hash), pass); err != nil {
return errors.Wrapf(err, "unable to verify hash for node '%s'", nodeName)
}
return nil
}
return fmt.Errorf("unable to locate hash data for node secret '%s'", name)
}
// Ensure will verify a node-password secret if it exists, otherwise it will create one
func Ensure(secretClient coreclient.SecretClient, nodeName, pass string) error {
if err := verifyHash(secretClient, nodeName, pass); !apierrors.IsNotFound(err) {
return err
}
hash, err := hasher.CreateHash(pass)
if err != nil {
return errors.Wrapf(err, "unable to create hash for node '%s'", nodeName)
}
immutable := true
_, err = secretClient.Create(&v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: getSecretName(nodeName),
Namespace: metav1.NamespaceSystem,
},
Immutable: &immutable,
Data: map[string][]byte{"hash": []byte(hash)},
})
if apierrors.IsAlreadyExists(err) {
return verifyHash(secretClient, nodeName, pass)
}
return err
}
// Delete will remove a node-password secret
func Delete(secretClient coreclient.SecretClient, nodeName string) error {
return secretClient.Delete(metav1.NamespaceSystem, getSecretName(nodeName), &metav1.DeleteOptions{})
}
// MigrateFile moves password file entries to secrets
func MigrateFile(secretClient coreclient.SecretClient, nodeClient coreclient.NodeClient, passwordFile string) error {
_, err := os.Stat(passwordFile)
if os.IsNotExist(err) {
return nil
}
if err != nil {
return err
}
passwd, err := passwd.Read(passwordFile)
if err != nil {
return err
}
nodeNames := []string{}
nodeList, _ := nodeClient.List(metav1.ListOptions{})
if nodeList != nil {
for _, node := range nodeList.Items {
nodeNames = append(nodeNames, node.Name)
}
}
if len(nodeNames) == 0 {
nodeNames = append(nodeNames, passwd.Users()...)
}
logrus.Infof("Migrating node password entries from '%s'", passwordFile)
ensured := int64(0)
start := time.Now()
for _, nodeName := range nodeNames {
if pass, ok := passwd.Pass(nodeName); ok {
if err := Ensure(secretClient, nodeName, pass); err != nil {
logrus.Warn(errors.Wrapf(err, "error migrating node password entry for node '%s'", nodeName))
} else {
ensured++
}
}
}
ms := time.Since(start).Milliseconds()
logrus.Infof("Migrated %d node password entries in %d milliseconds, average %d ms", ensured, ms, ms/ensured)
return os.Remove(passwordFile)
}
package nodepassword
import (
"fmt"
"io/ioutil"
"log"
"os"
"runtime"
"testing"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/watch"
)
const migrateNumNodes = 10
const createNumNodes = 3
func TestAsserts(t *testing.T) {
assertEqual(t, 1, 1)
assertNotEqual(t, 1, 0)
}
func TestEnsureDelete(t *testing.T) {
logMemUsage(t)
secretClient := &mockSecretClient{}
assertEqual(t, Ensure(secretClient, "node1", "Hello World"), nil)
assertEqual(t, Ensure(secretClient, "node1", "Hello World"), nil)
assertNotEqual(t, Ensure(secretClient, "node1", "Goodbye World"), nil)
assertEqual(t, secretClient.created, 1)
assertEqual(t, Delete(secretClient, "node1"), nil)
assertNotEqual(t, Delete(secretClient, "node1"), nil)
assertEqual(t, secretClient.deleted, 1)
assertEqual(t, Ensure(secretClient, "node1", "Hello Universe"), nil)
assertNotEqual(t, Ensure(secretClient, "node1", "Hello World"), nil)
assertEqual(t, Ensure(secretClient, "node1", "Hello Universe"), nil)
assertEqual(t, secretClient.created, 2)
logMemUsage(t)
}
func TestMigrateFile(t *testing.T) {
nodePasswordFile := generateNodePasswordFile(migrateNumNodes)
defer os.Remove(nodePasswordFile)
secretClient := &mockSecretClient{}
nodeClient := &mockNodeClient{}
logMemUsage(t)
if err := MigrateFile(secretClient, nodeClient, nodePasswordFile); err != nil {
log.Fatal(err)
}
logMemUsage(t)
assertEqual(t, secretClient.created, migrateNumNodes)
assertNotEqual(t, Ensure(secretClient, "node1", "Hello World"), nil)
assertEqual(t, Ensure(secretClient, "node1", "node1"), nil)
}
func TestMigrateFileNodes(t *testing.T) {
nodePasswordFile := generateNodePasswordFile(migrateNumNodes)
defer os.Remove(nodePasswordFile)
secretClient := &mockSecretClient{}
nodeClient := &mockNodeClient{}
nodeClient.nodes = make([]v1.Node, createNumNodes, createNumNodes)
for i := range nodeClient.nodes {
nodeClient.nodes[i].Name = fmt.Sprintf("node%d", i+1)
}
logMemUsage(t)
if err := MigrateFile(secretClient, nodeClient, nodePasswordFile); err != nil {
log.Fatal(err)
}
logMemUsage(t)
assertEqual(t, secretClient.created, createNumNodes)
for _, node := range nodeClient.nodes {
assertNotEqual(t, Ensure(secretClient, node.Name, "wrong-password"), nil)
assertEqual(t, Ensure(secretClient, node.Name, node.Name), nil)
}
newNode := fmt.Sprintf("node%d", createNumNodes+1)
assertEqual(t, Ensure(secretClient, newNode, "new-password"), nil)
assertNotEqual(t, Ensure(secretClient, newNode, "wrong-password"), nil)
}
// --------------------------
// mock secret client interface
type mockSecretClient struct {
entries map[string]map[string]v1.Secret
created int
deleted int
}
func (m *mockSecretClient) Create(secret *v1.Secret) (*v1.Secret, error) {
if m.entries == nil {
m.entries = map[string]map[string]v1.Secret{}
}
if _, ok := m.entries[secret.Namespace]; !ok {
m.entries[secret.Namespace] = map[string]v1.Secret{}
}
if _, ok := m.entries[secret.Namespace][secret.Name]; ok {
return nil, errorAlreadyExists()
}
m.created++
m.entries[secret.Namespace][secret.Name] = *secret
return secret, nil
}
func (m *mockSecretClient) Update(secret *v1.Secret) (*v1.Secret, error) {
return nil, errorNotImplemented()
}
func (m *mockSecretClient) Delete(namespace, name string, options *metav1.DeleteOptions) error {
if m.entries == nil {
return errorNotFound()
}
if _, ok := m.entries[namespace]; !ok {
return errorNotFound()
}
if _, ok := m.entries[namespace][name]; !ok {
return errorNotFound()
}
m.deleted++
delete(m.entries[namespace], name)
return nil
}
func (m *mockSecretClient) Get(namespace, name string, options metav1.GetOptions) (*v1.Secret, error) {
if m.entries == nil {
return nil, errorNotFound()
}
if _, ok := m.entries[namespace]; !ok {
return nil, errorNotFound()
}
if secret, ok := m.entries[namespace][name]; ok {
return &secret, nil
}
return nil, errorNotFound()
}
func (m *mockSecretClient) List(namespace string, opts metav1.ListOptions) (*v1.SecretList, error) {
return nil, errorNotImplemented()
}
func (m *mockSecretClient) Watch(namespace string, opts metav1.ListOptions) (watch.Interface, error) {
return nil, errorNotImplemented()
}
func (m *mockSecretClient) Patch(namespace, name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.Secret, err error) {
return nil, errorNotImplemented()
}
// --------------------------
// mock node client interface
type mockNodeClient struct {
nodes []v1.Node
}
func (m *mockNodeClient) Create(node *v1.Node) (*v1.Node, error) {
return nil, errorNotImplemented()
}
func (m *mockNodeClient) Update(node *v1.Node) (*v1.Node, error) {
return nil, errorNotImplemented()
}
func (m *mockNodeClient) UpdateStatus(node *v1.Node) (*v1.Node, error) {
return nil, errorNotImplemented()
}
func (m *mockNodeClient) Delete(name string, options *metav1.DeleteOptions) error {
return errorNotImplemented()
}
func (m *mockNodeClient) Get(name string, options metav1.GetOptions) (*v1.Node, error) {
return nil, errorNotImplemented()
}
func (m *mockNodeClient) List(opts metav1.ListOptions) (*v1.NodeList, error) {
return &v1.NodeList{Items: m.nodes}, nil
}
func (m *mockNodeClient) Watch(opts metav1.ListOptions) (watch.Interface, error) {
return nil, errorNotImplemented()
}
func (m *mockNodeClient) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.Node, err error) {
return nil, errorNotImplemented()
}
// --------------------------
// utility functions
func assertEqual(t *testing.T, a interface{}, b interface{}) {
if a != b {
t.Fatalf("[ %v != %v ]", a, b)
}
}
func assertNotEqual(t *testing.T, a interface{}, b interface{}) {
if a == b {
t.Fatalf("[ %v == %v ]", a, b)
}
}
func generateNodePasswordFile(migrateNumNodes int) string {
tempFile, err := ioutil.TempFile("", "node-password-test.*")