Harbor共享存储高可用

主机拓扑

角色 主机名 ip 系统 资源最低要求
Harbor1
nginx
Keepalived1		 harbor1 192.168.48.106 OpenEuler22.03LTS CPU：4核
内存：2G
硬盘：40G
Harbor2
nginx
Keepalived2		 harbor2 192.168.48.107 OpenEuler22.03LTS CPU：4核
内存：2G
硬盘：40G
postgresql
Redis
NFS共享		 zujian 192.168.48.108 OpenEuler22.03LTS CPU：4核
内存：2G
硬盘：40G
高可用ip 192.168.48.100

系统架构图

image-20240430185108978

基本配置

操作节点：[harbor1，harbour2，zujian]

1
vi jichu_init.sh

将以下脚本内容添加进去

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97

#!/bin/bash
if [ $# -eq 2 ];then
  echo "设置主机名为：$1"
  echo "ens160设置IP地址为：192.168.48.$2"
else
  echo  "使用方法：sh $0 主机名 主机位"
  exit 2
fi

echo "--------------------------------------"
echo "1.正在设置主机名：$1"
hostnamectl set-hostname $1

echo "2.正在关闭firewalld、dnsmasq、selinux"
systemctl disable firewalld &> /dev/null
systemctl disable dnsmasq &> /dev/null
systemctl stop firewalld
systemctl stop dnsmasq
sed -i "s#SELINUX=enforcing#SELINUX=disabled#g" /etc/selinux/config
setenforce 0


echo "3.正在设置ens160：192.168.48.$2"
cat > /etc/sysconfig/network-scripts/ifcfg-ens160 <<EOF
TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=static
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
NAME=ens160
UUID=53b402ff-5865-47dd-a853-7afcd6521738
DEVICE=ens160
ONBOOT=yes
IPADDR=192.168.48.$2
GATEWAY=192.168.48.2
PREFIX=24
DNS1=192.168.48.2
DNS2=114.114.114.114

nmcli c reload
nmcli c up ens 160

echo "4.优化ssh"
sed -i "s#\#UseDNS yes#UseDNS no#g" /etc/ssh/sshd_config
sed -i "s#GSSAPIAuthentication yes#GSSAPIAuthentication no#g" /etc/ssh/sshd_config
systemctl restart sshd

echo "5.更改欧拉源为华为云源，速度快一点"
sed -i 's/\$basearch/x86_64/g' /etc/yum.repos.d/openEuler.repo
sed -i 's/http\:\/\/repo.openeuler.org/https\:\/\/mirrors.huaweicloud.com\/openeuler/g' /etc/yum.repos.d/openEuler.repo


echo "6.更新yum源软件包缓存"
yum clean all && yum makecache
dnf update -y

echo "7.修改history格式及记录数"
sed -i "s#HISTSIZE=1000##g" /etc/profile
cat >> /etc/profile <<EOF
shopt -s histappend
USER_IP=`who -u am i 2>/dev/null| awk '{print $NF}'|sed -e 's/[()]//g'`
export HISTFILE=~/.commandline_warrior
export HISTTIMEFORMAT="%Y-%m-%d %H:%M:%S  `whoami`@${USER_IP}: "
export HISTSIZE=200000
export HISTFILESIZE=1000000
export PROMPT_COMMAND="history -a"
EOF
source /etc/profile


echo "8.添加hosts解析"
cat > /etc/hosts <<EOF
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.48.106 harbor1
192.168.48.107 harbor2
192.168.48.108 zujian
EOF


echo "10.安装chrony服务，并同步时间"
dnf install chrony -y
systemctl start chronyd
systemctl enable chronyd
chronyc sources
chronyc sources

echo "11、安装依赖包"
dnf install -y cmake gcc gcc-c++ perl readline readline-devel openssl openssl-devel zlib zlib-devel ncurses-devel readline readline-devel zlib zlib-devel

reboot
1
2
3
4
5
6
7
执行脚本命令格式：sh jichu_init.sh 主机名 主机位
[harbor1] sh jichu_init.sh harbor1 106

[harbor2] sh jichu_init.sh harbor2 107

[zujian] sh jichu_init.sh zujian 108

配置ssh免密

操作节点：[harbor1，harbour2，zujian]

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
dnf install -y sshpass 
cat > sshmianmi.sh << "EOF"
#!/bin/bash
# 目标主机列表
hosts=("harbor1" "harbor2" "zujian")
# 密码
password="Lj201840."
# 生成 SSH 密钥对
ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa

# 循环遍历目标主机
for host in "${hosts[@]}"
do
    # 复制公钥到目标主机
    sshpass -p "$password" ssh-copy-id -o StrictHostKeyChecking=no "$host"
    
    # 验证免密登录
    sshpass -p "$password" ssh -o StrictHostKeyChecking=no "$host" "echo '免密登录成功'"
done
EOF

sh sshmianmi.sh

安装高可用组件

操作节点:[harbor1，harbour2]

1
dnf install -y keepalived nginx

安装nginx

操作节点:[harbor1，harbour2]

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
cat > /etc/nginx/nginx.conf <<"EOF"
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log notice;
pid /run/nginx.pid;
include /usr/share/nginx/modules/*.conf;
events {
    worker_connections 1024;
}
stream {
    log_format  main  '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';
    access_log  /var/log/nginx/harbor-access.log  main;
    upstream harbor{
       server 192.168.48.106:8081;   #harbor1
       server 192.168.48.107:8081;   #harbor2
    }
    server {
       listen  80;
       proxy_pass harbor;
    }
}
http {
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';
    access_log  /var/log/nginx/access.log  main;
    sendfile            on;
    tcp_nopush          on;
    keepalive_timeout   65;
    types_hash_max_size 4096;
    include             /etc/nginx/mime.types;
    default_type        application/octet-stream;
    
    include /etc/nginx/conf.d/*.conf;
    server {
        listen       8888 default_server;
        server_name  _;
        location / {
        }
    }
}
EOF
systemctl enable --now nginx
nginx -s reload

安装安装keepalived

操作节点:[harbor1]

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
cat >/etc/keepalived/keepalived.conf << "EOF"
! Configuration File for keepalived

global_defs {
   notification_email {
     qianyios@qq.com
   }
   router_id harbor1
}

vrrp_instance zh {
    state MASTER
    interface ens160
    mcast_src_ip 192.168.48.106
    virtual_router_id 107
    priority 100
    advert_int 1
    nopreempt
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.48.100/24
    }
    track_script {
        chk_nginx
    }
}
vrrp_script chk_nginx {
    script "/etc/keepalived/check_nginx.sh"
    interval 2
    weight -20
}
EOF

操作节点:[harbor2]

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
cat >/etc/keepalived/keepalived.conf << "EOF"
! Configuration File for keepalived

global_defs {
   notification_email {
     qianyios@qq.com
   }
   router_id harbor2
}

vrrp_instance zh {
    state BACKUP
    interface ens160
    mcast_src_ip 192.168.48.107
    virtual_router_id 107
    priority 99
    advert_int 1
    nopreempt
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.48.100/24
    }
    track_script {
        chk_nginx
    }
}
vrrp_script chk_nginx {
    script "/etc/keepalived/check_nginx.sh"
    interval 2
    weight -20
}
EOF

配置检查脚本

操作节点:[harbor1，harbour2]

1
2
3
4
5
6
7
8
9
10
11
12
13
cat >/etc/keepalived/check_nginx.sh <<"EOF"
#!/bin/bash
counter=`ps -C nginx --no-header | wc -l`
if [ $counter -eq 0 ]; then
    systemctl start nginx
    sleep 2
    counter=`ps -C nginx --no-header | wc -l`
    if [ $counter -eq 0 ]; then
        systemctl stop keepalived
    fi
fi

EOF

启动服务

1
2
systemctl enable --now nginx keepalived
nginx -s reload

查看VIP虚拟ip

1
2
3
4
5
6
7
8
9
10
11
[root@harbor1 ~]# ip a
.............
2: ens160: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 00:0c:29:af:76:8c brd ff:ff:ff:ff:ff:ff
    inet 192.168.48.106/24 brd 192.168.48.255 scope global noprefixroute ens160
       valid_lft forever preferred_lft forever
    inet 192.168.48.100/24 scope global secondary ens160   ###192.168.48.100/24就是刚刚设置的高可用ip
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:feaf:768c/64 scope link noprefixroute
       valid_lft forever preferred_lft forever

安装postgresql

操作节点：[zujian]

安装

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
#创建postgres用户
useradd postgres
passwd postgres
#设置密码123456


#编译安装postgresql
wget https://ftp.postgresql.org/pub/source/v16.2/postgresql-16.2.tar.gz
tar zxvf postgresql-16.2.tar.gz -C /usr/local/bin/
cd /usr/local/bin/postgresql-16.2/
./configure --prefix=/usr/local/postgresql
make && make install

#建立数据目录
mkdir -p /data/postgresql/data
#创建日志目录
mkdir -p /data/postgresql/log
#创建socket目录
mkdir -p /data/postgresql/tmp
#授权
chown -R postgres:postgres /usr/local/postgresql/
chown -R postgres:postgres /data/postgresql

#设置postgres环境
su - postgres
cd
cat << "EOF" >> ~/.bash_profile
PGHOME=/usr/local/postgresql
export PGHOME
PGDATA=/data/postgresql/data
export PGDATA
PATH=$PATH:$HOME/bin:$HOME/.local/bin:$PGHOME/bin
export PATH
EOF
source ~/.bash_profile
psql -V

#初始化数据库
initdb --username=postgres -D /data/postgresql/data
#会有Success. You can now start the database server using:      #表示初始化成功


#修改初始化的配置文件
cat > /data/postgresql/data/postgresql.conf << "EOF"
max_connections = 100                   # 允许最大连接数
shared_buffers = 128MB                  # 内存大小
dynamic_shared_memory_type = posix      # the default is usually the first option
max_wal_size = 1GB
min_wal_size = 80MB
log_timezone = 'Asia/Shanghai'
datestyle = 'iso, mdy'
timezone = 'Asia/Shanghai'
lc_messages = 'en_US.UTF-8'             # locale for system error message
lc_monetary = 'en_US.UTF-8'             # locale for monetary formatting
lc_numeric = 'en_US.UTF-8'              # locale for number formatting
lc_time = 'en_US.UTF-8'                 # locale for time formatting
default_text_search_config = 'pg_catalog.english'
listen_addresses = '*' #监听所有地址
data_directory = '/data/postgresql/data'  # 数据目录指定
port = 5432
unix_socket_directories = '/data/postgresql/tmp'
unix_socket_group = ''
unix_socket_permissions = 0777
logging_collector = on
log_directory = '/data/postgresql/log'
log_rotation_size = 1GB
log_timezone = 'Asia/Shanghai'
log_min_duration_statement = 100

EOF
#设置远程连接
cat >> /data/postgresql/data/pg_hba.conf << EOF
local   all             all                                     trust
host    all             all             0.0.0.0/0               password
host    all             all             ::1/128                 password
host    all             postgres             0.0.0.0/0           trust
EOF


#启动PostgreSQL
pg_ctl -D /data/postgresql/data -l logfile start

进入数据库

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
psql -h 127.0.0.1 -p 5432 -U postgres

postgres=# \password
Enter new password for user "postgres":
Enter it again:
#输入密码123456
postgres=# exit

#重新启动
pg_ctl -D /data/postgresql/data -l /data/postgresql/data/postgresql.log restart

#提示一下信息成功（不是命令哈，不要去运行）
pg_ctl: old server process (PID: 25461) seems to be gone
starting server anyway
waiting for server to start.... done
server started

psql -h 127.0.0.1 -p 5432 -U postgres
#输入密码123456

CREATE DATABASE registry;
CREATE DATABASE notary_signer;
CREATE DATABASE notary_servers;
\l
create user server with password '123456';
create user signer with password '123456';
\du
GRANT ALL PRIVILEGES ON DATABASE registry to postgres;   
GRANT ALL PRIVILEGES ON DATABASE notary_signer to postgres;   
GRANT ALL PRIVILEGES ON DATABASE notary_servers to postgres;   
exit

image-20240421010122073

image-20240421010045648

设置启动服务

操作节点[zujian]

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
#回到root用户下执行
su - root
cat >/etc/init.d/PG-start.sh<< "EOF"
sudo -u postgres /usr/local/postgresql/bin/pg_ctl -D /data/postgresql/data start
EOF

cat >/etc/init.d/PG-stop.sh<< "EOF"
sudo -u postgres /usr/local/postgresql/bin/pg_ctl -D /data/postgresql/data stop
EOF

cat >/etc/init.d/PG-restart.sh<<"EOF"
sudo -u postgres /usr/local/postgresql/bin/pg_ctl -D /data/postgresql/data restart
EOF

sudo chmod +x /etc/init.d/PG-start.sh
sudo chmod +x /etc/init.d/PG-stop.sh
sudo chmod +x /etc/init.d/PG-restart.sh

cat >/etc/systemd/system/postgresql.service << "EOF"
[Unit]
Description=postgresql Service

[Service]
Type=oneshot
user=root
RemainAfterExit=true
ExecStart=/usr/bin/sudo /etc/init.d/PG-start.sh
ExecStop=/usr/bin/sudo /etc/init.d/PG-stop.sh
ExecRestart=/usr/bin/sudo /etc/init.d/PG-restart.sh

[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable --now postgresql

错误积累

1
2
3
4
5
6
7
8
#每次我启动pgsql的时候就有这个东西

2024-09-14 14:34:03.196 CST [17746] HINT:  Is another postmaster (PID 16042) running in data directory "/data/postgresql/data"?
 stopped waiting
pg_ctl: could not start server

#意思是有个pid进程在运行，杀掉它就行了
sudo kill -9 16042

安装redis

操作节点：[zujian]

安装redis

1
2
3
4
5
wget https://download.redis.io/releases/redis-7.2.4.tar.gz
tar zxvf redis-7.2.4.tar.gz 
mv redis-7.2.4 /usr/local/bin/
cd /usr/local/bin/redis-7.2.4
make && make install

修改配置文件

1
2
3
4
5
6
7
8
9
10
11
12
13
vi /usr/local/bin/redis-7.2.4/redis.conf

#bind 127.0.0.1 -::1  #注释掉bind的行，允许任何主机连接；
daemonize yes       #将no修改为yes，使redis可以使用守护进程方式启动；
requirepass 123456   #添加这行，设置redis连接的auth密码（123456）
protected-mode no  #禁用保护模式

以下是一步到位将以上四个命令全部实现
sed -i 's/^bind 127.0.0.1 -::1/#bind 127.0.0.1 -::1/' /usr/local/bin/redis-7.2.4/redis.conf
sed -i 's/^daemonize no/daemonize yes/' /usr/local/bin/redis-7.2.4/redis.conf
echo -e "\nrequirepass 123456" >> /usr/local/bin/redis-7.2.4/redis.conf
sed -i 's/^protected-mode yes/protected-mode no/' /usr/local/bin/redis-7.2.4/redis.conf

启动服务

1
redis-server redis.conf
1
2
[root@zujian redis-7.2.4]# redis-server redis.conf
2226:C 20 Apr 2024 17:10:45.039 # WARNING Memory overcommit must be enabled! Without it, a background save or replication may fail under low memory condition. Being disabled, it can also cause failures without low memory condition, see https://github.com/jemalloc/jemalloc/issues/1328. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect.

Redis在启动时可能会出现这样的日志：在分析这个问题之前， 首先要弄清楚什么是overcommit？ Linux操作系统对大部分申请内存的请求都回复yes， 以便能运行更多的程序。 因为申请内存后， 并不会马上使用内存， 这种技术叫做overcommit。如果Redis在启动时有上面的日志， 说明vm.overcommit_memory=0， Redis提示把它设置为1。
vm.overcommit_memory用来设置内存分配策略， 有三个可选值， 如表：可用内存代表物理内存与swap之和

image-20240420171525086

解决办法：

1
2
3
echo "vm.overcommit_memory=1" >> /etc/sysctl.conf
sysctl vm.overcommit_memory=1
redis-server redis.conf

再重新启动就可以查看版本和端口号了

1
2
3
4
5
6
[root@zujian redis-7.2.4]# redis-cli -v
redis-cli 7.2.4
[root@zujian redis-7.2.4]# ps aux |grep 6379
root        2227  0.0  0.3  68412 10888 ?        Ssl  17:10   0:00 redis-server *:6379
root        5427  0.0  0.0  22096  2300 pts/0    S+   17:19   0:00 grep --color=auto 6379
#redis-server有这个就行

关闭服务（只是普及知识，测试可用，不要随意关闭）

1
redis-cli shutdown

客户端连接redis

操作节点：[zujian]

将redis-cli的工具复制到Harbor1，harbor2

查看redis-cli工具位置

1
2
[root@zujian]# which redis-cli
/usr/local/bin/redis-cli

复制

1
2
3
which redis-cli
scp /usr/local/bin/redis-cli harbor1:/usr/local/bin/
scp /usr/local/bin/redis-cli harbor2:/usr/local/bin/

操作节点：[harbor1,harbor2]

1
redis-cli -h 192.168.48.108 -p 6379 -a 123456

image-20240420232245510

image-20240420232257725

到此redis安装成功

设置启动服务

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
cat >/etc/init.d/redis-start.sh<< "EOF"
/usr/local/bin/redis-server /usr/local/bin/redis-7.2.4/redis.conf
EOF
cat >/etc/init.d/redis-stop.sh<< "EOF"
/usr/local/bin/redis-cli -a 123456 shutdown
EOF

sudo chmod +x /etc/init.d/redis-start.sh
sudo chmod +x /etc/init.d/redis-stop.sh

cat >/etc/systemd/system/redis.service << "EOF"
[Unit]
Description=redis Service

[Service]
Type=oneshot
user=root
RemainAfterExit=true
ExecStart=/usr/bin/sudo /etc/init.d/redis-start.sh
ExecStop=/usr/bin/sudo /etc/init.d/redis-stop.sh
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable --now redis

NFS共享存储安装

操作节点：[zujian]

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
dnf install -y nfs-utils
systemctl enable --now nfs
#创建远程共享目录
mkdir -p /data/harbor_data

cat >> /etc/exports << "EOF"
/data/harbor_data 192.168.48.0/24(rw,no_root_squash)
EOF
#使配置生效
exportfs -arv

#生效结果
[root@zujian ~]# showmount -e
Export list for zujian:
/data/harbor_data 192.168.48.0/24

操作节点：[harbor1,harbor2]

Harbor1、harbor2机器上安装nfs-utils客户端并挂载共享存储

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
dnf install -y nfs-utils
systemctl enable --now nfs
mkdir -p /data/harbor_data
cat >>/etc/fstab<<"EOF"
192.168.48.108:/data/harbor_data /data/harbor_data nfs defaults 0 0
EOF
mount -a
df -h | grep harbor
#以下是挂载成功
[root@harbor1 ~]# df -h | grep harbor
192.168.48.108:/data/harbor_data   63G  3.0G   57G   6% /data/harbor_data

[root@harbor2 ~]# df -h | grep harbor
192.168.48.108:/data/harbor_data   63G  3.0G   57G   6% /data/harbor_data

harbor仓库安装

操作节点：[harbor1,harbor2]

安装docker

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
wget https://download.docker.com/linux/static/stable/x86_64/docker-26.0.1.tgz
tar xf docker-*.tgz
cp docker/* /usr/bin/
#创建containerd的service文件,并且启动
cat >/etc/systemd/system/containerd.service <<EOF
[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target local-fs.target
[Service]
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/bin/containerd
Type=notify
Delegate=yes
KillMode=process
Restart=always
RestartSec=5
LimitNPROC=infinity
LimitCORE=infinity
LimitNOFILE=1048576
TasksMax=infinity
OOMScoreAdjust=-999
[Install]
WantedBy=multi-user.target
EOF
systemctl enable --now containerd.service

#准备docker的service文件

cat > /etc/systemd/system/docker.service <<EOF
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service containerd.service
Wants=network-online.target
Requires=docker.socket containerd.service
[Service]
Type=notify
ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always
StartLimitBurst=3
StartLimitInterval=60s
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TasksMax=infinity
Delegate=yes
KillMode=process
OOMScoreAdjust=-500
[Install]
WantedBy=multi-user.target
EOF


#准备docker的socket文件

cat > /etc/systemd/system/docker.socket <<EOF
[Unit]
Description=Docker Socket for the API
[Socket]
ListenStream=/var/run/docker.sock
SocketMode=0660
SocketUser=root
SocketGroup=docker
[Install]
WantedBy=sockets.target
EOF
groupadd docker

systemctl enable --now docker.socket  && systemctl enable --now docker.service


#验证
mkdir /etc/docker
cat >/etc/docker/daemon.json <<EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "registry-mirrors": [
    "https://docker.mirrors.ustc.edu.cn",
    "http://hub-mirror.c.163.com",
    "https://pw860av8.mirror.aliyuncs.com"
  ],
  "max-concurrent-downloads": 10,
  "log-driver": "json-file",
  "log-level": "warn",
  "log-opts": {
    "max-size": "500m",
    "max-file": "3"
    },
  "data-root": "/var/lib/docker"
}
EOF
systemctl restart docker
docker -v

安装docker-compose

操作节点：[harbor1,harbor2]

1
2
3
4
wget https://github.com/docker/compose/releases/download/v2.26.1/docker-compose-linux-x86_64
mv docker-compose-linux-x86_64 /usr/local/bin/docker-compose
chmod +x /usr/local/bin/docker-compose
docker-compose version

配置内核参数并使之生效

操作节点：[harbor1,harbor2]

1
2
3
4
5
6
7
modprobe br_netfilter
cat >> /etc/sysctl.conf <<EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1 #路由转发
EOF
sysctl -p

下载harbor包并配置文件

操作节点：[harbor1,harbor2]

下载离线包offline字样

1
2
3
4
5
6
7
8
9
wget https://github.com/goharbor/harbor/releases/download/v2.9.4/harbor-offline-installer-v2.9.4.tgz
tar zxvf harbor-offline-installer-v2.9.4.tgz
mv harbor /var/
cd /var/harbor/

[root@harbor1 harbor]# ls
common.sh  harbor.v2.9.4.tar.gz  harbor.yml.tmpl  install.sh  LICENSE  prepare

cp harbor.yml.tmpl harbor.yml

配置harbor文件

操作节点：[harbor1]

1
vi /var/harbor/harbor.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
hostname: 192.168.48.106  #harbor1
http:
  port: 8081
  
#https:       #先注释https协议，后面再实现
 # port: 443
 # certificate: /your/certificate/path
 # private_key: /your/private/key/path

## 启用外部代理，启用后hostname将不再使用
external_url: https://192.168.48.100

#harbor页面密码
harbor_admin_password: Harbor12345


#配置NFS共享存储
data_volume: /data/harbor_data
_version: 2.9.0
#配置数据库
external_database:
  harbor:
    host: 192.168.48.108  # 数据库主机地址
    port: 5432              # 数据库端口
    db_name: registry    # 数据库名称
    username: postgres        # 连接该数据库的用户名
    password: 123456    # 连接数据库的密码
    ssl_mode: disable
    max_idle_conns: 50
    max_open_conns: 100
  notary_server:
    host: 192.168.48.108
    port: 5432
    db_name: notary_server
    username: postgres
    password: 123456
    ssl_mode: disable
  notary_signer:
    host: 192.168.48.108
    port: 5432
    db_name: notary_signer
    username: postgres
    password: 123456
    ssl_mode: disable 
#配置redis
external_redis:
  host: 192.168.48.108:6379 #redis服务IP地址和端口号
  password: 123456   #连接外部redis服务的密码
  registry_db_index: 1  
  jobservice_db_index: 2 #job服务的数据库索引
  chartmuseum_db_index: 3  #chartmuseum插件的Redis索引
  trivy_db_index: 5   #Trivy扫描器的数据索引
  idle_timeout_seconds: 30  #超时时间

#启用metrics数据采集插件
metric:
  enabled: false
  port: 9090
  path: /metrics

trivy:
  ignore_unfixed: false
  skip_update: false
  skip_java_db_update: false
  offline_scan: false
  security_check: vuln
  insecure: false
jobservice:
  max_job_workers: 10
  job_loggers:
    - STD_OUTPUT
    - FILE
  logger_sweeper_duration: 1 #days
notification:
  webhook_job_max_retry: 3
  webhook_job_http_client_timeout: 3 #seconds
log:
  level: info
  local:
    rotate_count: 50
    rotate_size: 200M
    location: /var/log/harbor
proxy:
  http_proxy:
  https_proxy:
  no_proxy:
  components:
    - core
    - jobservice
    - trivy
upload_purging:
  enabled: true
  age: 168h
  interval: 24h
  dryrun: false
cache:
  enabled: false
  expire_hours: 24

操作节点：[harbor2]

1
vi /var/harbor/harbor.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
hostname: 192.168.48.107  #harbor2
http:
  port: 8081
  
#https:       #先注释https协议，后面再实现
 # port: 443
 # certificate: /your/certificate/path
 # private_key: /your/private/key/path

## 启用外部代理，启用后hostname将不再使用
external_url: https://192.168.48.100

#harbor页面密码
harbor_admin_password: Harbor12345

#配置NFS共享存储
data_volume: /data/harbor_data
_version: 2.9.0
#配置数据库
external_database:
  harbor:
    host: 192.168.48.108  # 数据库主机地址
    port: 5432              # 数据库端口
    db_name: registry    # 数据库名称
    username: postgres        # 连接该数据库的用户名
    password: 123456    # 连接数据库的密码
    ssl_mode: disable
    max_idle_conns: 2
    max_open_conns: 0
notary_server:
  host: 192.168.48.108
  port: 5432
  db_name: notary_server
  username: postgres
  password: 123456
  ssl_mode: disable
notary_signer:
  host: 192.168.48.108
  port: 5432
  db_name: notary_signer
  username: postgres
  password: 123456
  ssl_mode: disable 
#配置redis
external_redis:
  host: 192.168.48.108:6379 #redis服务IP地址和端口号
  password: 123456   #连接外部redis服务的密码
  registry_db_index: 1  
  jobservice_db_index: 2 #job服务的数据库索引
  chartmuseum_db_index: 3  #chartmuseum插件的Redis索引
  trivy_db_index: 5   #Trivy扫描器的数据索引
  idle_timeout_seconds: 30  #超时时间

#启用metrics数据采集插件
metric:
  enabled: false
  port: 9090
  path: /metrics

trivy:
  ignore_unfixed: false
  skip_update: false
  skip_java_db_update: false
  offline_scan: false
  security_check: vuln
  insecure: false
jobservice:
  max_job_workers: 10
  job_loggers:
    - STD_OUTPUT
    - FILE
  logger_sweeper_duration: 1 #days
notification:
  webhook_job_max_retry: 3
  webhook_job_http_client_timeout: 3 #seconds
log:
  level: info
  local:
    rotate_count: 50
    rotate_size: 200M
    location: /var/log/harbor
proxy:
  http_proxy:
  https_proxy:
  no_proxy:
  components:
    - core
    - jobservice
    - trivy
upload_purging:
  enabled: true
  age: 168h
  interval: 24h
  dryrun: false
cache:
  enabled: false
  expire_hours: 24

将配置文件注入到各级件中并安装

先提前下载镜像吧，这里用博主构建的镜像速度会快一点

1
2
docker pull registry.cn-hangzhou.aliyuncs.com/qianyios/prepare:v2.9.4
docker tag registry.cn-hangzhou.aliyuncs.com/qianyios/prepare:v2.9.4 goharbor/prepare:v2.9.4

开始注入

1
2
cd /var/harbor/
./prepare

image-20240420230001756

开始安装

1
2
cd /var/harbor/
./install.sh

image-20240420230138644

配置自启动

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
cat >/usr/lib/systemd/system/harbor.service << "EOF"
[Unit]
Description=Harbor
After=docker.service systemd-networkd.service systemd-resolved.service nfs-server.service
Requires=docker.service
Documentation=http://github.com/vmware/harbor
[Service]
Type=simple
Restart=on-failure
RestartSec=5
ExecStart=/usr/local/bin/docker-compose -f /var/harbor/docker-compose.yml up
ExecStop=/usr/local/bin/docker-compose -f /var/harbor/docker-compose.yml down
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable harbor --now

大坑来了，之前找了两个星期都没解决的

到此，harbor安装完成，但是你在网页可能会出现不能用admin登入，会显示密码错误你需要进行下一步安装证书

原因：首先pg数据库在安装harbor时创建的admin是用sha256协议加密的

image-20240430182121148

而在我们harbor页面，我们并没有配置ssl证书，是http方式访问并没有sha256加密协议，意味着harbor再登入的时候，会出现密码错误，就是：网页登入验证===/===pg数据库验证，配置openssl证书，可以解决此问题,openssl包含sha256协议，这样就可以登入了。

OpenSSL是一个强大的加密库，广泛应用于互联网的各个角落，用于保护数据传输的安全。它实现了SSL和TLS协议，这些协议是现代网络安全的基石。

配置ssl证书

操作节点：[harbor1,harbor2]

生成ca证书

创建一个放置证书相关的目录，并使用cd进入该目录

1
2
3
4
5
6
7
8
mkdir /var/harbor/cert&& cd  /var/harbor/cert
## 1. 生成CA证书私钥
openssl genrsa -out ca.key 4096
## 2. 生成CA证书，可调整 -subj 选项来表明域名名称等信息
openssl req -x509 -new -nodes -sha512 -days 3650 \
 -subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=192.168.48.100" \
 -key ca.key \
 -out ca.crt

生成服务器证书

认证证书通常包含证书请求.csr文件、签名证书.crt文件及私钥.key文件，我这里harbor配置的hostname是192.168.48.100，所以最终需要生成192.168.48.100.crt、192.168.48.100.csr、192.168.48.100.key三个文件。

  • key：证书私钥，一般利用rsa等算法生成
  • csr：证书请求文件，利用证书私钥生成证书请求文件，该文件包含了服务器和地址等信息，申请人将该文件提交给CA机构，CA机构会根据该文件所携带的私钥信息来进行签名生成证书
  • crt：证书文件
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
## 1. 生成私钥
openssl genrsa -out 192.168.48.100.key 4096
## 2. 生成csr文件
openssl req -sha512 -new \
    -subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=192.168.48.100" \
    -key 192.168.48.100.key \
    -out 192.168.48.100.csr
## 3. 生成ssl匹配多域名文，例如既想使用域名又需要通过127.0.0.1本地地址登陆测试，可使用subjectAltName参数来进行配置
cat > v3.ext <<-EOF
authorityKeyIdentifier=keyid,issuer
basicConstraints=CA:FALSE
keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @alt_names

[alt_names]
DNS.1=192.168.48.100
DNS.2=127.0.0.1
IP.1=192.168.48.100
EOF
## 4. 根据v3.ext及csr文件请求生成crt证书文件
openssl x509 -req -sha512 -days 3650 \
    -extfile v3.ext \
    -CA ca.crt -CAkey ca.key -CAcreateserial \
    -in 192.168.48.100.csr \
    -out 192.168.48.100.crt

修改harbor配置文件

1
2
3
4
5
6
cat >> /var/harbor/harbor.yml << "EOF"
https:
  port: 443
  certificate: /var/harbor/cert/192.168.48.100.crt
  private_key: /var/harbor/cert/192.168.48.100.key
EOF

重新启动

1
2
3
4
cd /var/harbor
docker-compose down -v
./prepare
docker-compose up -d

镜像上传及拉取测试

操作节点：[harbor1,harbor2,zujian,qianyios（测试客户端）]

找一台客户端装好docker进行测试

1
2
[root@qianyios ~]# docker -v
Docker version 26.0.1, build d260a54

新建私有镜像仓库

image-20240429225926063

客户端免https登陆

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
# 此时直接使用docker login登陆到harbor中，会报错，下面hostname和port是harbor的配置文件中设置的名称及端口
#以下是格式
[root@xxxx harbor]# docker login [hostname]:[port]

可能会出现以下情况
#192.168.48.100是高可用vip
[root@harbor1 ~]# docker login 192.168.48.100
Username: admin
Password:
Error response from daemon: Get "https://192.168.48.100/v2/": tls: failed to verify certificate: x509: certificate signed by unknown authority

[root@qianyios ~]# docker login 192.168.48.106:8081
Username: admin
Password:
Error response from daemon: Get "https://192.168.48.106:8081/v2/": http: server gave HTTP response to HTTPS client
[root@qianyios ~]#

# 客户端默认使用的是https协议，所以需要对docker做以下修改,在文件末尾添加insecure-registries
[root@qianyios ~]# vim /etc/docker/daemon.json
{
   ................
  "registry-mirrors": [],#无关紧要，不用看,
  "insecure-registries": [ "192.168.48.100" ],#重要加这行，别忘了如果他不是最后一行一定要在末尾加逗号
 ................
}

# 修改后，重启docker使其生效
systemctl daemon-reload
systemctl restart docker

# 利用docker info查看是否添加上
[root@qianyios ~]# docker info
Containers: 10
 Running: 1
 Paused: 0
 Stopped: 9
Images: 37
...
 Experimental: false
 Insecure Registries:
  192.168.48.100   ###要确保有这个才行
  127.0.0.0/8
 Registry Mirrors:


扩展知识-containerd私有仓库配置（可略过）

在今后的K8s版本可能也会用containerd做为k8s的容器运行时，那么配置私有仓库也是一个头疼的事情。

在/etc/containerd/config.toml会有以下两个信息，可以定位

1
2
[plugins."io.containerd.grpc.v1.cri".registry.configs]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors]

要在以上两个信息下配置东西如下：（理解，我下面有一步到位命令，不用手动加）

1
2
3
4
5
6
7
8
9
10
11
12
[plugins."io.containerd.grpc.v1.cri".registry.configs]
  [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.48.100".tls]
    insecure_skip_verify = true  # 是否跳过安全认证
  [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.48.100".auth]
    username = "admin"
    password = "Harbor12345"

[plugins."io.containerd.grpc.v1.cri".registry.mirrors]
  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
    endpoint = ["https://registry-1.docker.io"]
  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."192.168.48.100"]
    endpoint = ["http://192.168.48.100"]

image-20241009163343160

添加Harbor信息

1
2
3
4
5
6
7
8
9
10
11
sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.configs\]/a \
        [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.48.100".tls]\
          insecure_skip_verify = true  # 是否跳过安全认证\
        [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.48.100".auth]\
          username = "admin"\
          password = "Harbor12345"' /etc/containerd/config.toml
sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a \
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]\
          endpoint = ["https://registry-1.docker.io"]\
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."192.168.48.100"]\
          endpoint = ["http://192.168.48.100"]' /etc/containerd/config.toml

最后尝试下载镜像

1
crictl pull 192.168.48.100/cicd/jenkins:latest

这个是我自己上传的镜像，已经在harbor仓库了，我现在在有containerd的客户端进行拉取看看能不能成功

image-20241009164535687

显然已经成功

进行登入测试

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
docker login 192.168.48.100

[root@harbor1 ~]# docker login 192.168.48.100
Username: admin
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store

Login Succeeded

[root@qianyios ~]# docker login 192.168.48.100
Username: admin
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store

Login Succeeded

##经过测试，通过添加"insecure-registries": [ "192.168.48.100" ]可以免除https登入

上传镜像测试

1
2
3
4
5
6
7
8
9
10
11
12
13
14
#下载一个nginx镜像，然后tag，再上传
[root@qianyios ~]# docker pull nginx
Using default tag: latest
latest: Pulling from library/nginx
a2abf6c4d29d: Pull complete
a9edb18cadd1: Pull complete
589b7251471a: Pull complete
186b1aaa4aa6: Pull complete
b4df32aa5a72: Pull complete
a0bcbecc962e: Pull complete
Digest: sha256:0d17b565c37bcbd895e9d92315a05c1c3c9a29f762b011a10c54a66cd53c9b31
Status: Downloaded newer image for nginx:latest
docker.io/library/nginx:latest
#在此下载了最新版的nginx镜像tag为lastest

我们进入当刚刚创建的仓库，点推送指令

1
2
3
#推送镜像命令格式
#docker tag 源镜像名[:TAG] 192.168.48.100/qianyios/新镜像名[:TAG]
docker tag SOURCE_IMAGE[:TAG] 192.168.48.100/qianyios/REPOSITORY[:TAG]

image-20240430180549537

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#我们将nginx镜像打上tag   
[root@qianyios ~]# docker images
REPOSITORY   TAG       IMAGE ID       CREATED       SIZE
nginx        latest    605c77e624dd   2 years ago   141MB
#605的意思是镜像id的前三位数字，我们指定为V1标签，相当于版本号。
[root@qianyios ~]# docker tag 605 192.168.48.100/qianyios/nginx:V1
[root@qianyios ~]# docker images
REPOSITORY                      TAG       IMAGE ID       CREATED       SIZE
192.168.48.100/qianyios/nginx   V1        605c77e624dd   2 years ago   141MB
nginx                           latest    605c77e624dd   2 years ago   141MB
#开始上传
[root@qianyios ~]# docker push 192.168.48.100/qianyios/nginx:V1
The push refers to repository [192.168.48.100/qianyios/nginx]
d874fd2bc83b: Pushed
32ce5f6a5106: Pushed
f1db227348d0: Pushed
b8d6e692a25e: Pushed
e379e8aedd4d: Pushed
2edcec3590a4: Pushed
V1: digest: sha256:ee89b00528ff4f02f2405e4ee221743ebc3f8e8dd0bfd5c4c20a2fa2aaa7ede3 size: 1570
#去页面查看

image-20240430181509733

image-20240430181528902

我们去harbor1测试拉取镜像，会发现下载数变成了1

1
2
3
4
5
6
7
8
9
10
11
12
[root@harbor1 ~]# docker pull 192.168.48.100/qianyios/nginx:V1
V1: Pulling from qianyios/nginx
a2abf6c4d29d: Pull complete
a9edb18cadd1: Pull complete
589b7251471a: Pull complete
186b1aaa4aa6: Pull complete
b4df32aa5a72: Pull complete
a0bcbecc962e: Pull complete
Digest: sha256:ee89b00528ff4f02f2405e4ee221743ebc3f8e8dd0bfd5c4c20a2fa2aaa7ede3
Status: Downloaded newer image for 192.168.48.100/qianyios/nginx:V1
192.168.48.100/qianyios/nginx:V1
[root@harbor1 ~]#

image-20240430181626605

特别声明
千屹博客旗下的所有文章，是通过本人课堂学习和课外自学所精心整理的知识巨著
难免会有出错的地方
如果细心的你发现了小失误，可以在下方评论区告诉我，或者私信我！
非常感谢大家的热烈支持！