Docker容器运行时引擎-runC分析

docker pull busybox
mkdir mycontainer && cd mycontainer
mkdir rootfs
# 创建rootfs，基于busybox镜像
docker export $(docker create busybox) | tar -C rootfs -xvf -
# 根据OCI规范来生成配置文件
runc spec

# 1. 修改config文件中的process元素的terminal与args
"process": {
		"terminal": false,
		"args": [
			"/bin/sleep", "3600"
		]
# 2. runc create创建容器（此时状态为created,容器内进程为init进程）
sudo runc create mycontainer
# 3. runc start启动容器（此时状态为running，容器内进程为我们指定的进程）
sudo runc start mycontainer
# 4. runc 暂停/恢复容器（暂停时状态为pause）
sudo runc pause mycontainer
sudo runc resume mycontainer
# 5. runc kill关闭容器（此时状态为stopped）
sudo runc kill mycontainer
# 6. runc delete删除容器
sudo runc delete mycontainer
# 7. runc run组合命令，组合了runc create创建，runc start启动以及在退出之后runc delete的删除命令
sudo runc run mycontainer

$ tree -L 1 -F --dirsfirst
.
├── contrib/
├── docs/
├── libcontainer/
├── man/
├── script/
├── tests/
├── types/
├── vendor/
├── checkpoint.go
├── create.go
├── delete.go
├── events.go
├── exec.go
├── init.go
├── kill.go
├── list.go
├── main.go
├── pause.go
├── ps.go
├── restore.go
├── run.go
├── spec.go
├── start.go
├── state.go
...

Action: func(context *cli.Context) error {
		if err := checkArgs(context, 1, exactArgs); err != nil {
			return err
		}
		if err := revisePidFile(context); err != nil {
			return err
		}
		spec, err := setupSpec(context)
		if err != nil {
			return err
		}
		status, err := startContainer(context, spec, CT_ACT_RUN, nil)
		if err == nil {
			os.Exit(status)
		}
		return err
	}

func startContainer(context *cli.Context, spec *specs.Spec, action CtAct, criuOpts *libcontainer.CriuOpts) (int, error) {
	id := context.Args().First()
	...
	container, err := createContainer(context, id, spec)
	if err != nil {
		return -1, err
	}
	...
	r := &runner{
		enableSubreaper: !context.Bool("no-subreaper"),
		shouldDestroy:   true,
		container:       container,
		listenFDs:       listenFDs,
		notifySocket:    notifySocket,
		consoleSocket:   context.String("console-socket"),
		detach:          context.Bool("detach"),
		pidFile:         context.String("pid-file"),
		preserveFDs:     context.Int("preserve-fds"),
		action:          action,
		criuOpts:        criuOpts,
		init:            true,
		logLevel:        logLevel,
	}
	return r.run(spec.Process)
}

func createContainer(context *cli.Context, id string, spec *specs.Spec) (libcontainer.Container, error) {
	...
    config, err := specconv.CreateLibcontainerConfig(&specconv.CreateOpts{
		CgroupName:       id,
		UseSystemdCgroup: context.GlobalBool("systemd-cgroup"),
		NoPivotRoot:      context.Bool("no-pivot"),
		NoNewKeyring:     context.Bool("no-new-keyring"),
		Spec:             spec,
		RootlessEUID:     os.Geteuid() != 0,
		RootlessCgroups:  rootlessCg,
	})
	if err != nil {
		return nil, err
	}

	factory, err := loadFactory(context)
	if err != nil {
		return nil, err
	}
	return factory.Create(id, config)
}

func (l *LinuxFactory) Create(id string, config *configs.Config) (Container, error) {
	...
	c := &linuxContainer{
		id:            id,
		root:          containerRoot,
		config:        config,
		initPath:      l.InitPath,
		initArgs:      l.InitArgs,
		criuPath:      l.CriuPath,
		newuidmapPath: l.NewuidmapPath,
		newgidmapPath: l.NewgidmapPath,
		cgroupManager: l.NewCgroupsManager(config.Cgroups, nil),
	}
	if l.NewIntelRdtManager != nil {
		c.intelRdtManager = l.NewIntelRdtManager(config, id, "")
	}
	c.state = &stoppedState{c: c}
	return c, nil
}

func New(root string, options ...func(*LinuxFactory) error) (Factory, error) {
	...
	l := &LinuxFactory{
		Root:      root,
		InitPath:  "/proc/self/exe",
		InitArgs:  []string{os.Args[0], "init"},
		Validator: validate.New(),
		CriuPath:  "criu",
	}
	Cgroupfs(l)
	for _, opt := range options {
		if opt == nil {
			continue
		}
		if err := opt(l); err != nil {
			return nil, err
		}
	}
	return l, nil
}

func (c *linuxContainer) start(process *Process) (retErr error) {
	parent, err := c.newParentProcess(process)
	if err != nil {
		return newSystemErrorWithCause(err, "creating new parent process")
	}
    //
	if err := parent.start(); err != nil {
		return newSystemErrorWithCause(err, "starting container process")
	}
	...
	//更新容器状态，PostStart Hook回调
	return nil
}

func (c *linuxContainer) newParentProcess(p *Process) (parentProcess, error) {
	parentInitPipe, childInitPipe, err := utils.NewSockPair("init")
	if err != nil {
		return nil, newSystemErrorWithCause(err, "creating new init pipe")
	}
	messageSockPair := filePair{parentInitPipe, childInitPipe}

	parentLogPipe, childLogPipe, err := os.Pipe()
	if err != nil {
		return nil, fmt.Errorf("Unable to create the log pipe:  %s", err)
	}
	logFilePair := filePair{parentLogPipe, childLogPipe}

	cmd := c.commandTemplate(p, childInitPipe, childLogPipe)
	if !p.Init {
		return c.newSetnsProcess(p, cmd, messageSockPair, logFilePair)
	}
	if err := c.includeExecFifo(cmd); err != nil {
		return nil, newSystemErrorWithCause(err, "including execfifo in cmd.Exec setup")
	}
	return c.newInitProcess(p, cmd, messageSockPair, logFilePair)
}

func (p *initProcess) start() (retErr error) {
	defer p.messageSockPair.parent.Close()
    //启动容器init进程
	err := p.cmd.Start()
	p.process.ops = p
	p.messageSockPair.child.Close()
	p.logFilePair.child.Close()
    waitInit := initWaiter(p.messageSockPair.parent)
	...
    //将容器pid加入到cgroup 中
	if err := p.manager.Apply(p.pid()); err != nil {
		return newSystemErrorWithCause(err, "applying cgroup configuration for process")
	}
	if p.intelRdtManager != nil {
		if err := p.intelRdtManager.Apply(p.pid()); err != nil {
			return newSystemErrorWithCause(err, "applying Intel RDT configuration for process")
		}
	}
    //向容器init进程发送bootstrapData，即初始化数据
	if _, err := io.Copy(p.messageSockPair.parent, p.bootstrapData); err != nil {
		return newSystemErrorWithCause(err, "copying bootstrap data to pipe")
	}
    //等待容器init进程初始化
	err = <-waitInit
	if err != nil {
		return err
	}
	...
	//创建网络 interface
	if err := p.createNetworkInterfaces(); err != nil {
		return newSystemErrorWithCause(err, "creating network interfaces")
	}
    //更新Spec状态
	if err := p.updateSpecState(); err != nil {
		return newSystemErrorWithCause(err, "updating the spec state")
	}
    // 给容器init进程发送进程配置信息
	if err := p.sendConfig(); err != nil {
		return newSystemErrorWithCause(err, "sending config to init process")
	}
	//和容器init进程进行同步,待其初始化完成之后，执行Hook回调操作，设置cgroup，最后，关闭init管道，容器创建完成
	ierr := parseSync(p.messageSockPair.parent, func(sync *syncT) error {
    ...
    }
	...
}

func (c *linuxContainer) exec() error {
	path := filepath.Join(c.root, execFifoFilename)
	pid := c.initProcess.pid()
	blockingFifoOpenCh := awaitFifoOpen(path)
	for {
		select {
		case result := <-blockingFifoOpenCh:
			return handleFifoResult(result)

		case <-time.After(time.Millisecond * 100):
			stat, err := system.Stat(pid)
			if err != nil || stat.State == system.Zombie {
				if err := handleFifoResult(fifoOpen(path, false)); err != nil {
					return errors.New("container process is already dead")
				}
				return nil
			}
		}
	}
}

func (l *linuxStandardInit) Init() error {

	//配置容器网络与路由
	if err := setupNetwork(l.config); err != nil {
		return err
	}
	if err := setupRoute(l.config.Config); err != nil {
		return err
	}

	//设置rootfs，挂载文件系统，chroot
	if err := prepareRootfs(l.pipe, l.config); err != nil {
		return err
	}
	//配置console, hostname, apparmor, sysctl，seccomp，user namespace...
	...
	//同步父进程 init进程准备执行exec
	if err := syncParentReady(l.pipe); err != nil {
		return errors.Wrap(err, "sync ready")
	}
	...
	//已完成初始化，关闭管道
	l.pipe.Close()
	...
	//向exec.fifo管道写入数据，阻塞到exec执行，读取管道中的数据
	fd, err := unix.Open("/proc/self/fd/"+strconv.Itoa(l.fifoFd), unix.O_WRONLY|unix.O_CLOEXEC, 0)
	if err != nil {
		return newSystemErrorWithCause(err, "open exec fifo")
	}
	if _, err := unix.Write(fd, []byte("0")); err != nil {
		return newSystemErrorWithCause(err, "write 0 exec fifo")
	}
	//调用Exec系统命令，执行用户进程
	if err := unix.Exec(name, l.config.Args[0:], os.Environ()); err != nil {
		return newSystemErrorWithCause(err, "exec user process")
	}
	return nil
}