linux下http协议 多线程下载实现.doc

linux下http协议 多线程下载实现该程序只能在http协议下工作，等以后研究其他协议时再补充。- E ?! K+ Z9 y2 5 K2 s* r! a j( I, G. EJ- F( % M, P. (网络服务器用的是腾讯公司的linuxqq 希望他们能原谅我)3 b, W0 ?* k V% 9 p- 编程思路：2 x& s; ?8 b! : zq! r- V1、分析http协议 数据包。6 I, h- O. P2 N R9 R- G1 Z以下是一个请求报文与相应的回复报文的例子, j+ ?) l0 / i; y9 P, S, YE- h. j o1 2 e. GET /linuxqq/linuxqq-v1.0.2-beta1.i386.rpm HTTP/1.1, ? q8 Z2 r; l( v9 RAccept: */*8 L0 G: I9 y, g n( TAccept-Language: en-us* R9 v7 q1 m: - p9 FUser-Agent: Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)$ zK! 5 X FA: h* w) i8 jHost: :80H8 ; p- G) e) R/ w8 ) 9 HConnection: Keep-Alive# t2 k3 X/ ?& c- c5 C. Z. F0 J2 X* _; P/ ) 9 HTTP/1.1 206 Partial Content: U* c4 w: u! w v! c, HContent-Length: 1009349) l4 q: j4 / EKContent-Range: bytes 0-1009348/5046743; q4 C5 V3 O$ c& G% U/ Server: qqdlsrv(1.84 for linux). q& r3 X* O d+ U* J7 iXConnection: Keep-alive3 F; bh6 S/ f, x2 w3 |Content-Disposition: attachment; filename=linuxqq-v1.0.2-beta1.i386.rpm% ?. C9 X# ?9 k. U0 y* i8 Z$ cAccept-Ranges: bytes# o t0 p3 q4 K1 c! Y6 jContent-Type: application/octet-stream9 K, y k& E) f6 H$ z$ 7 h0 Q; r. K: b9 m$ e1 ?$ H$ D, G; k; U2、取得文件大小，然后进行对文件的分块。) T1 * r1 I3 Z2 3、对文件分块下载, S% d: m1 J7 M, s. U4、合并文件，并改名。 ! y) 3 d% L+ D5 B& A( w) L9 P8 p, wq$ v+ A, ; w6 B总结：这个分块再合并总感觉太过于啰嗦。6 A0 P3 L+ W E. r, v5 u是否还有另外一种方法，7 0 % p% p t& r1 Y# R; v( 如迅雷，在一开始就取得文件大小就在硬盘分配空间，6 DM0 % E+ C C5 m) c% E然后分块下载直接写入到文件的不同块？2 A i2 g6 ?9 G0 i是否可以用文件指针锁住不同的区域并写入？ y5 s8 n5 L, a% z8 bF9 Y: |; | t# q: x; O8 9 M# * A& 9 u% P7 x: c, L! V3 v/*# w: B- m* p2 I# k6 S+ 3 + r* down.c2 d2 W8 H4 G3 C9 w*! g$ c, D0 E, w6 f, j7 P! w*Created on: Mar 27, 20093 S( x- L% j5 l9 o- u* Author: root: i& ! ?. J# B% h! J C* this program is to download file in http protocol with multthreading( v& d) e! q; : y# r! |8 K5 Wi*/0 . k. i& f- E. p0 o0 r9 U$ L6 Z#ifndef STRUCT_H_% h& 9 v2 q2 3 J/ b#define STRUCT_H_3 U3 f% w$ i: B2 Q( d% KRJ, * Y0 u! m1 j! ?8 G#endif /* STRUCT_H_ */5 t0 N4 H T* k& r& _3 E8 V#include ! y U9 o9 i F- p#include & h9 a, 3 z2 o: p8 I) x4 H! y5 K#include . 1 ?$ z6 n# W+ a#include & F. F3 q9 K* D G#include 2 z( o! + |, #include 6 P+ + g6 I7 k9 t4 z3 v#include K# s: p7 e) 9 C: N V#include & z4 z& z! K& F! z/create a struct to save the file part infomation8 r3 O/ f% zW0 8 O5 : w4 Tstruct filepart& P9 T+ r* N+ E4 X( Z5 M$ P1 r3 q0 3 E! ?2 o+ N( d: B; S% O: g5 Gint partnum;) - e# u6 G k+ k- y7 fchar pathname128;! p- q; f: h1 b, M0 P% 5 ilong int lenth;0 u8 w0 , Q2 nlong int datastar;$ $ L- x6 L$ T. |) % jlong int dataend;6 k6 el7 F! |; Y kstruct filepart *next;% m% K$ l! O Z5 Q6 % L;4 h% : M7 C8 ctypedef struct filepart FilePart;0 E! ) G, : 9 Pstruct hostent *host;9 A: L, Y* * g: Fchar filepath2048;/save the filepath on the serverP% E0 c% 8 t: w mchar hostname1024;/save the name of the server3 w# $ _. Y4 r# Z9 g: O( _char filename256;/save the name of the file# W# F/ b& E( # ochar localpath512; 3 |- h; r7 8 C- Mint port,part;+ B) N) V* # x8 C, T7 blong int FileLen;/the file lenth0 d, w+ B; B7 T! S- + d/ u j9 Nvoid getname( char * ulr); /解析url，并从中取得主机名，ip地址 、文件路径以及端口0 8 m/ m+ _* P5 yvoid filepart( long int datalen,int part ); /对文件进行分块6 C! V& a# W6 v3 lvoid *getfile( FilePart *pointer); /下载线程函数3 e Y# H, K# qo% T% q$ A7 dlong int getFileLen( struct hostent *host); /获取http包头，分析文件大小1 ( n/ K6 N4 U. b+ r; i5 7 , vvoid createthread( FilePart * head, int num );/创建线程函数 参数为链表头指针和分块数目: 4 A- l7 8 ?2 A* int mergefile( FilePart * head);0 o 2 u! v. a. 4 9 Hint main(void) A9 2 Q7 Y+ 4 F, B% 4 , g# W& Q* G, % h8 j4 r Jchar ulr128;4 Z v: t; W* j+ P eprintf(please input the website:n); Q H& v. t, x6 dscanf(%s,ulr);- _+ L( P1 H4 nprintf(the website is :%sn,ulr);2 / h% ! f. Fx uprintf(input the direct to save file:n);( X O! ?$ Z+ U/ ; Z jscanf(%s,localpath);* j; 2 r: z: Z8 I Dgetname( ulr ); /get the ipaddress of the host8 m$ Z: k7 S; J. z/ getfile( host );+ |2 l9 ) e8 - E. o0 m) ?s aFileLen = getFileLen( host ); /get the lenth of the file9 S3 m) m6 D- PZ; S* N( + Ffilepart( FileLen, 3);8 0 Q- E* T, T6 k# ) |8 C2 I4 8 O M* a4 Tputs(!Hello World!); /* prints !Hello World! */6 , 0 T; k4 P1 c Sreturn EXIT_SUCCESS;, W7 b4 R4 F% X( E ( A- C N1 s7 |+ N8 H3 |( Z1 z6 w7 ?0 B* v7 k# q, f- . P+ Z: ; y/ K* Z# O! V. u% F. M( G1 Xlong int getFileLen( struct hostent *host)0 8 i9 * A; c- i! t, B+ T3 J9 - J0 3 o. mstruct sockaddr_in sock_serv;O% x3 p7 E5 bK u, |int sockfd;9 Z; b& P# 6 X( Dchar Request1024;4 AR% a* b% S- q. ylong int filelen;1 b. l; w5 m7 w K! K1 L9 1 Q) Lint recvbytes;* y! W% Y t8 0 _, 2 v/ zchar *buff ,*buffp,*Plen;! o; N4 f& Z, h* Ibuff = (char *)malloc( 1024 * sizeof(char);/ W1 U6 yo& F memset( buff, 0, sizeof(buff);0 z; r E& , dsockfd = socket( AF_INET, SOCK_STREAM,0);Z3 R7 L0 ?1 f e- z, T/ printf(sockfd = %dn,sockfd);$ y+ f- ; Z# L4 D+ 6 I4 jif(sockfd h_addr); e! Z4 H% 4 q9 r8 p! Sif(0connect(sockfd,(struct sockaddr *)&sock_serv, sizeof(struct sockaddr)0 f! j9 u% t- a6 r& T: u 9 z0 Z* R6 B& |perror(connect);. n; i+ m, Z% T z) j) sexit(1);2 B; r2 2 E8 - q/ O4 w& & ? S/ O5 L; w1 j6 E$ i8 ! I% C/ printf(connect success!n); j; V* a2 o7 U* WGprintf(file path is:%sn,filepath);! A& Q* h+ . $ _/ % zD5 zsprintf(Request, GET %s HTTP/1.1rnAccept: */*rnAccept-Language: en-usrnUser-Agent: Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)rnHost: %s:%drnConnection: Keep-Alivernrn, filepath, hostname, port);9 g/ D+ T2 b1 T) k, O; T/ printf(%s,Request);6 k3 y0 u T2 J0 E$ B! R# N( h) P# V* Y/send the http protocl packet/ o: i1 6 q+ wif( 0 send(sockfd, Request, strlen(Request), 0)6 Y0 m1 . F7 b5 v0 i8 b- H b( ii; O# ?perror(send);7 Z- ( R& b$ V3 o! |exit(1);9 m+ d( - p9 c0 u2 r1 + u* ?+ o0 D5 D, L0 gwhile( 1 )4 v& G) Z1 c7 q B: s# p( _: N4 s, m* Y! h0 3 L9 d5 y# 7 H6 Jrecvbytes = recv(sockfd, buff, 1024,0);2 N$ o* pq/ x* q w/printf(recvbytes = %d,recvbytes);, K- P: E; . H) / Q/ : Bif(0 recvbytes )! p- , Y2 i. T! TW) B( I, u s- e8 Y0 X, eS/ z M6 C4 U q; perror(recv);7 ; T( K; R5 x& K exit(1);4 _: * ?8 Y9 Z* ox! J6 u+ 5 e/ G6 N v- V 5 EJPlen = strstr(buff, Length:);. 3 + K5 0 lif( NULL != Plen )9 j( K3 p; g3 z$ L T7 p9 % K0 2 v; d r# G* I Plen = strchr( Plen, :);8 z1 I$ c6 8 T: B Plen+;8 ?2 T $ f% 2 r& - _ filelen = atol( Plen );: C * o4 i0 T5 Y8 s) . G5 u- X printf(the file data is %ldn,filelen);) h8 L9 $ 4 C0 R/ n0 X& 3 n0 / buffp = strstr(buff, rnrn);: e) N% C2 m. /printf(n%s,buff);1 F& R: D5 M3 - Dif( NULL != buffp ), R3 ?/ C$ f# E9 o/ l/ 9 a break;- q/ X$ 9 l( H6 Dmm1 t1 z4 R5 X: M M5 Vfree( buff ); buff = NULL;0 |0 s( k9 G* p; O& ) Yclose(sockfd);& P1 b9 y2 e* W7 lreturn filelen;( ?! b( n6 f9 tq7 A4 L. ?+ M/ m1 P% n$ A) h0 Q& ?# j, j2 g: Y+ 2 + B2 e4 dvoid getname( char * ulr) u, V5 Z$ A6 m$ c9 v* F V) xchar head10; /save thehttp:/ or https:/- s( _& M1 B! z2 W$ kchar hostname1024;, d2 j0 k. |4 t% b* q0 Xchar *pointer;. | / 0 v V) R; e/ L+ rchar *pointer2;0 g! Y% b/ U p: Z6 E5 wchar *addrs;1 j k& I& ( ( + T9 R2 Opointer = strstr(ulr,/); /move the pointer to the / of the url& e; g6 ? Z* C9 z! ypointer += 2; /move the pointer to the right of / of the urlA& V4 s8 D* Pstrncpy(head, ulr, pointer - ulr);/copy 8 / r: j6 x2 r) shttp:/ to head0 d* 1 _. 4 H( K! / printf(head = %sn,head);% g# X5 G0 K) L# N7 U2 Qdpointer2 = strchr( pointer,:); /find the port of the server1 d+ d$ Y0 _2 p6 a, jif( pointer2 = NULL) /if there is no port( 4 $ e# m4 t3 % Y W* 7 C% z( Bchar *temp = strchr(pointer,/); /move pointer to the first /5 Y& d R; H4 v& D7 p/ 7 T- $ hport = 80;: w. . 2 m+ d+ Q( K* _if( temp != NULL )2 m K9 X+ a M! Ev5 L9 M# T# h/ n3 e% f! A pointer2 = temp+; /save a pointer pointo the right of the first /$ b5 + D# m/ D8 X0 B1 6 U# strncpy(hostname, pointer, pointer2 - pointer);/save the host name. h M K7 g& / | strcpy(filepath, pointer2); /the right of the first / are the filepath and name, then save!$ t6 q$ 3 T+ e! o9 y5 i0 printf(filepath:%sn,filepath);( i( O7 s+ I h; k7 g9 R0 p while( NULL != temp )0 q$ $ r! u6 E+ q, _ 1 w9 6 C8 t( temp = strchr( temp,/); /find the right /* E w7 S) X2 X Z& S if( NULL != temp )/ x! ?lL3 ?4 o% G: j6 d + n e& D0 L5 X b$ G3 D. e- C4 T pointer2 = temp;1 E* k+ b# 4 m3 f: - F temp+;- j j0 K; 3 C$ M& Z/ F 5 x2 O3 k Y G$ Z. a* 1 s& f 7 A+ f) Z/ C, strcpy(filename, +pointer2); /the right of the right / is the file name! y. ; n6 l- ?* B; A. U printf(filename is: %sn,filename);0 X2 : B+ |% b% c /between the first left / and the right / is the file path5 : $ r: / f printf(save to %sn,localpath); F5 _0 |* 0 a) i; T# j. /* if( *temp = 0 )+ , B5 5 Z2 8 y. U m7 5 ?9 _+ C8 U6 X 8 O; c4 W6 ?: _4 K* 1 s strcpy(filepath,/index.html);1 T1 3 K m4 G printf(filepath:%sn,filepath);6 M/ w% W3 E4 l. T J x& |$ o- H4 w, L*/- e e, o y|A! k3 J+ T+ U h. o$ X; i. xelse1 K+ QS) u* S; u r. * R+ l4 l( A1 W strcpy(hostname,pointer);# V% 5 a* p: N w8 3 R7 t strcpy(filepath,/index.html);- $ k5 G o, z! s7 w o printf(filepath = %sn,filepath);1 Rb0 Q5 c% u8 G* : M y( n- o: c6 u/ m5 j2 q# S8 r% G/ x% h* delse z4 J5 g( M) L5 T( P2 z ( ; D/ q6 f/ L H strncpy(hostname, pointer, pointer2 - pointer);/ r, M% M( Z4 w5 X6 r. u: y port = atoi(+pointer2);% Y* M. F9 s- E/ J0 E2 printf(port = %dn,port);, h7 g7 u R. r% - Q3 l! K/ j, : F8 V$ K! y% H3 Nprintf(hostname = %sn,hostname);4 |5 K j: U* M: bhost = gethostbyname(hostname); v! B! x; s* W! g, Bif( host h_addr_list;R! y2 i; c 2 T$ Zprintf(%s,inet_ntoa(*(struct in_addr *)*addrs);- j+ J7 x# 3 U# Xprintf(n);/ n* |% J2 C: L+ L3 i& U+ i( E7 N( 0 K, d% - R6 A1 k( U9 O* h6 T+ B6 p, F$ p$ Zvoid filepart( long int datalen,int part ) /move the file lenth and the number of cutting parts; G# 9 V 3 n* I/ I4 D A# W1 h$ g3 _3 * S( rlong int partlen = datalen / part;, k6 a; % 6 V; long int end = datalen - 1;. u1 W: T5 s. j- O; y% |, Jint i;3 R3 b6 e* J i* . R8 D: Q% Y/ printf(partlen = %ldnend = %ldn,partlen,end);3 K0 E, C( j) x* I& s; ! FilePart *head ,*present;$ U, B- O+ e) z. 3 j/ ; head = (FilePart *)malloc(sizeof(FilePart); /create the head of the link/ |- Oj& P$ M) ?: Khead - partnum =1;- # Y3 a. g# - W0 Y3 Y1 Z# Bhead - datastar = 0;) + f, g) . S, ) M( |4 G+ x, Hhead - dataend = partlen;* & R4 ?# h# d6 C. % g4 % k! Ehead - lenth = partlen;# G3 c0 c I0 Z0 ( C0 psprintf(head - pathname,%s/part%d,localpath,1);8 H3 Ad0 - P s+ I! x- a! qif(part = 1); b 2 h% q# U0 j3 4 ihead - dataend = end; v1 Y9 m. R5 $ G: F: U0 Ypresent = head;! A; A2 h8 t/ 2 F, f! lfor( i = 0; i next = (FilePart *)malloc(sizeof(FilePart);2 7 m: f+ e n1 y7 A. Lmemset(present-next, 0, sizeof(FilePart);6 r# W1 H0 m& c6 T! xpresent -next - partnum = i+2;8 b$ _2 & N) a$ xpresent -next - datastar = present - dataend +1;( v- D. 9 7 g- v9 j, dif( i = (part-2), ; # 5 ?. s% s * A1 t& J% w/ s4 ! h present -next - dataend = end;2 z4 d2 K u# j3 C present -next - lenth = datalen - (part-1)*partlen; W2 9 X6 A% I, H0 m& L 8 c& 1 U* O+ t: u; R8 p, telse _) U V2 E: Q3 U! k% s4 R/ I & _$ A3 ?- H y7 A present -next - dataend = partlen * (i+2);f5 e1 Y) k7 x- f present -next - lenth = partlen;7 5 & , i- q ? 2 1 G F+ n2 4 9 e8 Isprintf(present -next-pathname,%s/part%d,localpath,(i+2);) D8 W: a, a0 V) |5 Lpresent = present-next;& y1 M0 - aQ4 j0 i( B* i! - y! Y+ g8 7 6 i& J) Ccreatethread( head, part );! f 3 Z: F7 I: umergefile( head );: k h# l! |$ I* 3 . e; t+ W4 s6 w. y$ gvoid createthread( FilePart * head, int num )7 V; m% Z6 n( j5 M3 O8 v P1 f1 Y) $ b# P) s% S0 j/ printf(start to create threadn);6 U* s/ C g) T ; h* |: int i = 1 ,res;- B6 h7 J; R. R8 a$ L/ M% blong int presnum;9 x( XF% q( x- c: vpthread_t threadnum;- k9 k# r V: O. v& f: s& WFilePart *present = head;9 s! W3 A) W9 N. G1 B3 q0 for(i = 0; i next;) D9 jc/ m7 d1 p1 m7 S& z- I, z9 8 E& a) D; ! long int all = 0;- D5 m7 v. P! Ufor( i = 0; i num; i+) | W/ e) g1 A: , y( U( t0 l) x7 R7 / 3 y1 t. |! res = pthread_join(thread,&pres);4 - V& _9 o6 o5 x* j2 nif( res != 0), i2 $ V1 C/ a( 9 7 lm S2 - f$ R; r perror(pthread join);8 N! P4 |9 o- o4 b o3 g5 E& c exit(1); O- y) k4 g) D3 c( B5 q; c7 g3 ?printf(thread %d finished!n,i+1);2 w& x2 u# 8 D2 Qall += pres;% R1 i7 C0 l( Y8 t* b1 |2 ; Z0 . 1 k P( 9 x0 Oprintf(all download:%ld!n,all);4 L* b |! u$ N( I1 i) Q3 h/ l( p) ?: t5 $ x5 Evoid *getfile( FilePart *pointer)3 E) G7 + z$ B/ . U3 K. x4 x- ; W* E; sJ H# W- Hchar buff1024,request2048;4 m+ g+ 2 H7 U/ b* n+ F0 hstruct sockaddr_in sock_serv;* I9 M8 M$ O0 L3 NFILE *fp,* fplog;, d k0 J3 c; 7 F6 k- K3 Cint recvbytes,psockfd;$ i8 b/ F% m* c, T2 F5 u, Rlong int download = 0; 3 t; Y) E* ! P$ F# j$ KE rpsockfd = socket(AF_INET, SOCK_STREAM, 0);* a6 P( e- f5 f% vif( psockfd h_addr); U+ D, O3 | iD9 z- gif( ( connect( psockfd, (struct sockaddr *)&sock_serv, sizeof(struct sockaddr) ) )datastar,pointer -dataend);( Spv: W5 , B# L2 t/ printf(ready to send:n %s,request);* I* ) s! z! h) ?4 J1 ) B% Hint sends = send( psockfd, request, strlen(request), 0);# m X1 A6 _9 I% u; bif( sends partnum);! h0 + W0 blfplog = fopen( filelog,w+);8 f+ g6 f: 8 gfp = fopen( pointer-pathname,w+);5 M5 . N: Z% Tif(fp 0 )8 e% ! j# n# Z; P1 h- a6 G$ L$ qv t% : i- S% N# e3 Bperror(open );9 J+ u( k: O7 ipthread_exit(1);) E8 C; VX. 1 p+ w( |! q V) e. ?; Ointin = 0;$ a0 |# pG9 ( b7 l. g Qdo j L9 M a3 K/ j4 8 |3 O/ K. h x5 X! Imemset(buff, 0, sizeof(buff);+ x& VQ. V0 Precvbytes =