cha13.文件I/O缓冲

13.1

使用shell内嵌的time命令,测算程序清单4-1(copy.c)在当前环境下的用时。
a)使用不同的文件和缓冲区大小进行试验。编译应用程序时使用
-DBUF_SIZE=nbytes选项可设置缓冲区大小。
b) 对open()的系统调用加入O_SYNC标识,针对不同大小的缓冲区,速度存在多
大差异?
c) 在一系列文件系统(比如,ext3、XFS、Btrfs和 JFS)中执行这些计时测试。结果相似吗?当缓冲区大小从小变大时,用时趋势相同吗?

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>
#include <stdarg.h>
#include <stdio.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>

#ifndef BUF_SIZE
#define BUF_SIZE 1024
#endif

#define ERR(code, format, ...) do { \
if(!(code)) { \
fprintf(stderr, (char*)format, ##__VA_ARGS__); \
exit(code); \
} \
} while(0)

int main(int argc, char *argv[]) {
char buf[BUF_SIZE];
ssize_t readsize = 0;
int openflag = O_RDONLY;
#ifdef SYNC
openflag |= O_SYNC;
#endif
int inputfd = open(argv[1], openflag);
ERR(inputfd != -1, "fail to open %s, err:%s\n", argv[1], strerror(errno));

int outputfd = open(argv[2],
O_CREAT | O_WRONLY | O_TRUNC,
S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
ERR(outputfd != -1, "fail to open %s, err:%s\n", argv[2], strerror(errno));

while((readsize = read(inputfd, buf, BUF_SIZE)) > 0) {
ERR(write(outputfd, buf, readsize) == readsize, "could not write whole buffer, err:%s\n", strerror(errno));
}
ERR(readsize != -1, "read fail, err:%s", strerror(errno));
ERR(close(inputfd) != -1, "fail to close input, err:%s\n", strerror(errno));
ERR(close(outputfd) != -1, "fail to close output, err:%s\n", strerror(errno));
return 0;
}

测试时间

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
BUFSIZE=1
cat /dev/null > log13.1.log
for i in `seq 15`; do
echo round $i, BUFSIZE=$BUFSIZE >> log13.1.log
gcc practice13_1.c -DBUF_SIZE=$BUFSIZE -o practice13_1
/usr/bin/time -f "real = %e\nuser = %U\nsystem = %S" -o log13.1.log -a ./practice13_1 big big.copy
BUFSIZE=`expr $BUFSIZE \* 2`
done

BUFSIZE=1
cat /dev/null > log13.1_sync.log
for i in `seq 15`; do
echo round $i, BUFSIZE=$BUFSIZE >> log13.1_sync.log
gcc practice13_1.c -DBUF_SIZE=$BUFSIZE -DSYNC -o practice13_1
/usr/bin/time -f "real = %e\nuser = %U\nsystem = %S" -o log13.1_sync.log -a ./practice13_1 big big.copy
BUFSIZE=`expr $BUFSIZE \* 2`
done

生成md

阅读更多

cha12.系统和进程信息

12.1

编写一个程序,以用户名作为命令行参数,列表显示该用户下所有正在运行的进程ID和命令名。(程序清单8-1中的userldFromName()函数对本题程序的编写可能会有所帮助。)通过分析系统中/proc/PID/status文件的 Name:和 Uid:各行信息,可以实现此功能。遍历系统的所有/proc/PID目录需要使用readdir(3)函数,18.8节对其进行了描述。程序必须能够正确处理如下可能性:在确定目录存在与程序尝试打开相应/proc/PID/status文件之间,/proc/PID目录消失了。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
#include <unistd.h>
#include <dirent.h>
#include <stdio.h>
#include <string.h>
#include <pwd.h>
#include <errno.h>
#include <stdlib.h>
#include <sys/types.h>
#include <unistd.h>
#include <limits.h>

uid_t getUid(const char * user) {
errno = 0;
struct passwd *ret = getpwnam(user);
if(ret == NULL) {
fprintf(stderr, "ERROR: fail to get uid of user '%s'\n", user);
exit(1);
}
return ret->pw_uid;
}

int get_pid_max() {
FILE *pid_max = fopen("/proc/sys/kernel/pid_max", "r");
int ret = -1;
fscanf(pid_max, "%d", &ret);
fclose(pid_max);
if(ret == -1) {
fprintf(stderr, "Error: fail to read /proc/sys/kernel/pid_max\n");
exit(3);
}
return ret;
}

int main(int argc, char **argv) {
if(argc < 2 || strcmp(argv[1], "-help") == 0) {
fprintf(stderr, "Usage: user list [-help]\n");
exit(0);
}
pid_t pid_max = get_pid_max();
uid_t *uidlist = (uid_t *)alloca(argc * sizeof(uid_t));
pid_t **uid2pids = (pid_t **)alloca(argc * sizeof(pid_t*));
for(int i = 1; i < argc; i++) {
uidlist[i] = getUid(argv[i]);
uid2pids[i] = (pid_t *)alloca((pid_max + 1) * sizeof(pid_t));
uid2pids[i][0] = 0;
}
DIR *proc = opendir("/proc");
if(proc == NULL) {
fprintf(stderr, "ERROR: fail to read /proc: %s\n", strerror(errno));
exit(1);
}
struct dirent *proc_rent = NULL;
while((proc_rent = readdir(proc)) != NULL) {
const char *spid = proc_rent->d_name;
char *end;
errno = 0;
pid_t pid = strtol(spid, &end, 10);
if(end == spid || *end != '\0' || errno != 0) {
fprintf(stderr, "INFO: %s/%s is not a pid\n", "/proc", spid);
continue;
}
char filename[128] = {0};
sprintf(filename, "/proc/%s/status", spid);
FILE *status = fopen(filename, "r");
if(status == NULL) {
fprintf(stderr, "ERROR: fail to open %s\n", filename);
exit(4);
}
uid_t realUid = -1;
char buffer[1024] = {0};
fscanf(status, "%*[^\n]\n");
fscanf(status, "%*[^\n]\n");
fscanf(status, "%*[^\n]\n");
fscanf(status, "%*[^\n]\n");
fscanf(status, "%*[^\n]\n");
fscanf(status, "%*[^\n]\n");
fscanf(status, "%*[^\n]\n");
fscanf(status, "%*[^\n]\n");
fscanf(status, "%*s %u", &realUid); //忽略前8行
if(realUid == -1) {
fprintf(stderr, "ERROR: fail to read Uid in %s/%s/status\n", "/proc", spid);
exit(2);
}
for(int i = 1; i < argc; i++) {
if(realUid == uidlist[i]) {
uid2pids[i][++uid2pids[i][0]] = pid;
break;
}
}
fclose(status);
}
for(int i = 1; i < argc; i++) {
printf("---------------Process of User: %s, uid = %u---------------\n", argv[i], uidlist[i]);
for(int j = 1; j < uid2pids[i][0]; j++) {
printf("\t├ %d\n", uid2pids[i][j]);
}
if(uid2pids[i][0] == 0) {
printf("\t(nil)\n");
} else {
printf("\t└ %d\n", uid2pids[i][uid2pids[i][0]]);
}
}
closedir(proc);
}

/proc/PID目录消失

我觉得不要去读/proc/PID目录就好了,直接读/proc/PID/status,不存在就返回NULL, 然后读取下一个pid

12.2

阅读更多

cha9.进程凭证

9.1

9-1.在下列每种情况中,假设进程用户ID的初始值分别为real(实际) = 1000、effective(有效)= 0、saved(保存)= 0、file-system(文件系统)= 0。当执行这些调用后,用户ID的状态如何?

1
2
3
4
5
setuid(2000);
setreuid(-1, 2000);
seteuid(2000);
setfsuid(2000);
setresuid(-1,2000,3000);

实验代码

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
#define _GNU_SOURCE
#include <unistd.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <sys/fsuid.h>

int main(int argc, char *argv[]) {
setresuid(1000, 0, 0);
setfsuid(0);
int test = atoi(argv[1]);
uid_t r, e, s, fs;
if(getresuid(&r, &e, &s) == -1) return 1;
fs = setfsuid(0);
printf("real = %u, effective = %u, save = %u, fs = %u\n", r, e, s, fs);
switch (test) {
case 1:
printf("setuid(2000) = %d\n", setuid(2000));
break;
case 2:
printf("setreuid(-1, 2000) = %d\n", setreuid(-1, 2000));
break;
case 3:
printf("seteuid(2000) = %d\n", seteuid(2000));
break;
case 4:
printf("setfsuid(2000) = %d\n", setfsuid(2000));
break;
case 5:
printf("setresuid(-1, 2000, 3000) = %d\n", setresuid(-1, 2000, 3000));
break;
}
if(getresuid(&r, &e, &s) == -1) return 1;
fs = setfsuid(0);
printf("real = %u, effective = %u, save = %u, fs = %u\n", r, e, s, fs);
return 0;
}

环境准备

阅读更多

cha8.用户和组

8.1

运行下面代码,为什么输出会相同?

1
2
3
4
5
6
7
8
9
10
#include <stdio.h>
#include <unistd.h>
#include <pwd.h>
int main() {
struct passwd * p1 = getpwnam("redis");
struct passwd * p2 = getpwnam("sshd");
printf("getpwnam(\"redis\")->pw_uid = %u, getpwnam(\"sshd\")->pw_uid = %u\n", p1->pw_uid, p2->pw_uid);
printf("getpwnam(\"redis\") = %p, getpwnam(\"sshd\") = %p\n", p1, p2);
return 0;
}

getpwnam和getpwuid返回的指针指向由静态分配的的内存,地址都是相同的,所以会导致相同。

8.2

用getpwent,setpwent,endpwent实现getpwnam

阅读更多

cha7.内存分配

7.1

修改程序清单7-1中的程序(free_and_sbrk.c),在每次执行malloc后打印出 program break的当前值。指定一个较小的内存分配尺寸来运行该程序。这将证明malloc不会在每次被调用时都调用sbrk()来调整program break 的位置,而是周期性地分配大块内存,并从中将小片内存返回给调用者。

1
// 与代码7.2main相同

7.2

(高级)实现 malloc()和 free()。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <string.h>

#define CHECK(x, code, message, ...) if(!(x)) {fprintf(stderr, "%s:%d, error: %s\t----\t", __FILE__, __LINE__, strerror(errno)); fprintf(stderr, (const char*)message, ##__VA_ARGS__); exit(code); }
#define ERR(code, format, ...) fprintf(stderr, (const char*)format, ##__VA_ARGS__); exit(code)

#define UNUSED 0
#define USED 1

#define EXIT_SBRK_FAIL 1
#define UNKNOWN_FAIL 2
#define UNKNOWN_MEM_ERROR 3
#define FREE_TWICE 4

#define MAXALLOC 100000

void *freeMem = NULL;

unsigned long getBlockSize(void *mem) {
return *((unsigned long *)mem - 1);
}

void setBlockSize(void *mem, size_t size) {
*((unsigned long *)mem - 1) = size;
}

void *getNextFreeBlock(void *__free) {
return (unsigned long*) *((unsigned long *)__free + 1);
}

void setNextFreeBlock(void *__free, void *__ptr) {
*((unsigned long *)__free + 1) = (unsigned long)__ptr;
}

void *getPrevFreeBlock(void *__free) {
return (unsigned long*) *(unsigned long *)__free;
}

void setPrevFreeBlock(void *__free, void *__ptr) {
*(unsigned long *)__free = (unsigned long)__ptr;
}

char getUsed(void *__ptr) {
return *((char *)__ptr - 1 - sizeof(void *));
}

void setUsed(void *__ptr, char used) {
*((char *)__ptr - 1 - sizeof(void *)) = used;
}

void memInit(char used, void *__ptr, void *prev, void *next, size_t size) {
setUsed(__ptr, used);
setBlockSize(__ptr, size);
setPrevFreeBlock(__ptr, prev);
setNextFreeBlock(__ptr, next);
}

void *firstFit(size_t size) {
void *move = freeMem;
void *next = getNextFreeBlock(freeMem);
while(next != NULL) {
if(getBlockSize(next) >= size) {
break;
}
move = next;
next = getNextFreeBlock(next);
}
return move;
}

void *__malloc(size_t size) {
if(freeMem == NULL) {
freeMem = sbrk(sizeof(void *) * 3 + 1);
CHECK(freeMem != (void*)-1, EXIT_SBRK_FAIL, "sbrk fail\n");
freeMem += sizeof(void *) + 1;
memInit(UNUSED, freeMem, NULL, NULL, sizeof(void *) * 2);
}
void *__free = firstFit(size);
void *newMem = NULL;
CHECK(__free != NULL, UNKNOWN_FAIL, "unknown error\n");
if(getNextFreeBlock(__free) == NULL) {
size = size > 2 * sizeof(void *) ? size : 2 * sizeof(void *);
newMem = sbrk(1 + sizeof(void *) + size);
CHECK(newMem != (void*)-1, EXIT_SBRK_FAIL, "sbrk fail\n");
newMem += 1 + sizeof(void *);
memInit(USED, newMem, NULL, NULL, size);
} else {
newMem = getNextFreeBlock(__free);
setUsed(newMem, USED);
setNextFreeBlock(__free, getNextFreeBlock(newMem));
}
return newMem;
}

void __free(void * __ptr) {
CHECK(freeMem != NULL, UNKNOWN_MEM_ERROR,"memory: %p is not allocated by __mallo\n", __ptr);
CHECK(getUsed(__ptr) == USED, FREE_TWICE, "trying to free memory %p twice\n", __ptr);
setUsed(__ptr, UNUSED);
void *move = freeMem;
void *next = getNextFreeBlock(freeMem);
void *front = (char *)__ptr - 1 - sizeof(void *), *back = (char *)__ptr + getBlockSize(__ptr);
while(next != NULL) {
if((char *)next - 1 - sizeof(void *) >= (char *)back) {
break;
}
move = next;
next = getNextFreeBlock(next);
}
if(front == (char *)move + getBlockSize(move)) {
setBlockSize(move, (char *)back - (char *)move);
__ptr = move;
} else {
setNextFreeBlock(__ptr, getNextFreeBlock(move));
setNextFreeBlock(move, __ptr);
setPrevFreeBlock(__ptr, move);
}
if(next == NULL) return;
if(back == (char *)next - 1 - sizeof(void *)) {
setBlockSize(__ptr, (char *)next + getBlockSize(next) - (char *)__ptr);
} else {
setPrevFreeBlock(next, __ptr);
}
return;
}

void __printMemblock(void* ptr) {
printf("------------Memory Block %p---------------\n", ptr);
printf("front = %p, back = %p\n", (char *)ptr - 1 - sizeof(void *), (char *)ptr + getBlockSize(ptr));
int used = *((char *)ptr - 1 - sizeof(unsigned long));
printf("used\t\t=\t%d\n", used);
printf("blocksize\t=\t%lu\n", *((unsigned long *)ptr - 1));
if(!used) {
printf("last free block\t=\t%p\n", (unsigned long*) *(unsigned long *)ptr);
printf("next free block\t=\t%p\n", (unsigned long*) *((unsigned long *)ptr + 1));
}
printf("Current brk = %p\n", sbrk(0));
}

void __showFreeBlocks() {
printf("show free blocks\n");
void *move = freeMem;
while(move) {
__printMemblock(move);
move = getNextFreeBlock(move);
}
}

int main(int argc, char *argv[]) {
if(argc < 3) {
ERR(1, "Usage: %s numalloc blocksize [freestep] [freemin] [freemax]\n", argv[0]);
}
int numalloc = atoi(argv[1]);
size_t blocksize = atoi(argv[2]);
int freestep = argc > 3 ? atoi(argv[3]) : 1;
int freemin = argc > 4 ? atoi(argv[4]) : 1;
int freemax = argc > 5 ? atoi(argv[5]) : numalloc;

void *ptr[MAXALLOC];

if(numalloc > MAXALLOC) {
ERR(2, "constraint: numalloc <= %d\n", MAXALLOC);
}

printf("sizeof(void *) = %lu\n", sizeof(void *));
printf("Start to allocate mem, Current program break: %p\n", sbrk(0));

for(int i = 0; i < numalloc; i++) {
ptr[i] = __malloc(blocksize);
if(ptr[i] == NULL) {
ERR(3, "fail to __malloc loc: %d\n", i);
}
printf("Current program break: %p\n", sbrk(0));
__printMemblock(ptr[i]);
}

for(int i = 0; i < numalloc; i++) {
__printMemblock(ptr[i]);
}
__showFreeBlocks();
printf("Allocation finished, Current program break: %p\n", sbrk(0));

for(int i = freemin-1; i < freemax; i += freestep) {
__free(ptr[i]);
printf("Current program break: %p\n", sbrk(0));
__printMemblock(ptr[i]);
}

printf("Mem __free finished, Current program break: %p\n", sbrk(0));

for(int i = freemin-1; i < freemax; i += freestep) {
__printMemblock(ptr[i]);
}
__showFreeBlocks();
return 0;
}
阅读更多

cha6.进程

练习1

编译程序清单6-1中的程序(mem_segments.c),使用1s-l命令显示可执行文件的大小。虽然该程序包含一个大约10MB的数组,但可执行文件大小要远小于此,为什么?

  • 局部变量,分配在栈中,运行时分配

练习2

编写一个程序,观察当使用 longjmp()函数跳转到一个已经返回的函数时会发生什么?

  • 开优化会无限递归,不开优化也有可能无限递归
阅读更多

cha5.深入探究文件IO

练习1

请使用标准文件IO系统调用(open()和lseek())和off_t数据类型修改程序清单5-3中的程序。将宏_FILE_OFFSET_BITS的值设置为64进行编译,并测试该程序是否能够成功创建一个大文件。

将xxx64改为xxx,off64_t改为off_t,可以创建大文件(使用 -m32编译)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
// #define _LARGEFILE64_SOURCE
#define _FILE_OFFSET_BITS 64
#include<string.h>
#include<sys/stat.h>
#include<fcntl.h>
#include<unistd.h>
#include<errno.h>
#include<stdlib.h>

#define debug
#ifdef debug
#include<stdio.h>
#endif
void writeErr(const char* str);

#ifdef _LARGEFILE64_SOURCE
int main(int argc, char **argv) {
#ifdef debug
printf("sizeof(long) = %d, sizeof(long long) = %d, sizeof(off_t) = %d, sizeof(off64_t) = %d\n", sizeof(long), sizeof(long long), sizeof(off_t), sizeof(off64_t));
#endif
if(argc !=3 || strcmp(argv[1], "--help") == 0) {
writeErr(argv[0]);
writeErr(" pathname offset\n");
exit(3);
}
int fd = open64(argv[1], O_RDWR | O_CREAT, S_IRUSR|S_IWUSR);
if(fd == -1) {
writeErr("open64 fail");
exit(2);
}
off64_t off = atoll(argv[2]);
if(lseek64(fd, off, SEEK_SET) == -1) {
writeErr("lseek64");
exit(3);
}
if(write(fd, "test", 4) == -1) {
writeErr("write");
exit(4);
}
return 0;
}
#endif

#ifdef _FILE_OFFSET_BITS
int main(int argc, char **argv) {
#ifdef debug
printf("sizeof(long) = %d, sizeof(long long) = %d, sizeof(off_t) = %d\n", sizeof(long), sizeof(long long), sizeof(off_t));
#endif
if(argc !=3 || strcmp(argv[1], "--help") == 0) {
writeErr(argv[0]);
writeErr(" pathname offset\n");
exit(3);
}
int fd = open(argv[1], O_RDWR | O_CREAT, S_IRUSR|S_IWUSR);
if(fd == -1) {
writeErr("open64 fail");
exit(2);
}
off_t off = atoll(argv[2]);
if(lseek(fd, off, SEEK_SET) == -1) {
writeErr("lseek64");
exit(3);
}
if(write(fd, "test", 4) == -1) {
writeErr("write");
exit(4);
}
return 0;
}

#endif
void writeErr(const char* str) {
errno = 0;
int writeSize = write(STDERR_FILENO, str, strlen(str));
if(writeSize == -1) {
exit(1);
}
}

练习2

5-2.编写一个程序,使用O_APPEND标志并以写方式打开一个已存在的文件,且将文件偏移量置于文件起始处,再写入数据。数据会显示在文件中的哪个位置?为什么?

阅读更多

cha4.文件IO:通用的I/O模型

实现tee

tee命令是从标准输入中读取数据,直至文件结尾,随后将数据写入标准输出和命令行参数所指定的文件。(44.7节讨论FIFO时,会展示使用tee命令的一个例子。)请使用IO系统调用实现tee命令。默认情况下,若已存在与命令行参数指定文件同名的文件,tee命令会将其覆盖。如文件已存在,请实现-a命令行选项tee-a file在文件结尾处追加数据。(请参考附录B中对getopt)函数的描述来解析命令行选项。

预处理与函数声明

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
#include<string.h>

#include<sys/stat.h>
#include<fcntl.h>
#include<unistd.h>
#include<stdlib.h>

#include<errno.h> // 使用变量errno

#define bool char
#define false 0
#define true 1
#define strcmp(a,b) strcmp(a,b)==0

// exit code
#define UNSUPPORTED_ARG 1
#define FILE_OPEN_FAIL 2
#define STDOUT_WRITE_FAIL 3
#define STDERR_WRITE_FAIL 4
#define STDIN_READ_FAIL 5
#define PARTIAL_WRITE_OCCURED 6
#define FILE_CLOSE_FAIL 7
#define FILE_WRITE_FAIL 8

// 函数声明
void printHelp();
void printVersion();
void writeErr(const char* str);
void writeStdout(const char* str);
void exitErr(const char * err, int exitcode);
void release();

// 常量
#define MAX_FILE_COUNT 100
#define READ_BUFFER_SIZE 100

// 全局资源
char **files = NULL;
int *fds = NULL;
int fileCount = 0;

不引用c的库函数,直接使用系统调用对文件进行读写,不引入stdio.h

工具函数

阅读更多