netfilter-nf-conntrack-模块实现分析.doc

一连接记录的存储相关函数：static inline struct nf_conn *nf_ct_tuplehash_to_ctrack(const struct nf_conntrack_tuple_hash *hash)static inline struct nf_conn *nf_ct_get(const struct sk_buff *skb, enum ip_conntrack_info *ctinfo)nf_conntrack中tuple存储如上图所示。在struct nf_conntrack_tuple_hash中，成员hnode链接入ct-hash-first中的。实际的记录在保存在struct nf_conntrack_tuple中。1.记录的访问：hlist_nulls_for_each_entry_rcu 103 #define hlist_nulls_for_each_entry_rcu(tpos, pos, head, member) 104 for (pos = rcu_dereference(head)-first); 105 (!is_a_nulls(pos) & 106 ( tpos = hlist_nulls_entry(pos, typeof(*tpos), member); 1; ); 107 pos = rcu_dereference(pos-next) 108 参见 /net/netfilter/nf_conntrack_core.c _nf_conntrack_find函数；struct nf_conntrack_tuple_hash *h;struct hlist_nulls_node *n;hlist_nulls_for_each_entry_rcu(h, n, &net-ct.hashhash, hnnode)2.记录的添加 84 static inline void hlist_nulls_add_head_rcu(struct hlist_nulls_node *n, 85 struct hlist_nulls_head *h) 86 87 struct hlist_nulls_node *first = h-first; 88 89 n-next = first; 90 n-pprev = &h-first; 91 rcu_assign_pointer(h-first, n); 92 if (!is_a_nulls(first) 93 first-pprev = &n-next; 94 参见/net/netfilter/nf_conntrack_core.c： 313 static void _nf_conntrack_hash_insert(struct nf_conn *ct, 314 unsigned int hash, 315 unsigned int repl_hash) 316 317 struct net *net = nf_ct_net(ct); 318 319 hlist_nulls_add_head_rcu(&ct-tuplehashIP_CT_DIR_ORIGINAL.hnnode, 320 &net-ct.hashhash); 321 hlist_nulls_add_head_rcu(&ct-tuplehashIP_CT_DIR_REPLY.hnnode, 322 &net-ct.hashrepl_hash); 323 3.记录的删除 59 static inline void hlist_nulls_del_rcu(struct hlist_nulls_node *n) 60 61 _hlist_nulls_del(n); 62 n-pprev = LIST_POISON2; 63 59 static inline void _hlist_nulls_del(struct hlist_nulls_node *n) 60 61 struct hlist_nulls_node *next = n-next; 62 struct hlist_nulls_node *pprev = n-pprev; 63 *pprev = next; 64 if (!is_a_nulls(next) 65 next-pprev = pprev; 66 二nf_conntrack模块的初始化/net/netfilter/nf_conntrack_standalone.c510 static int _init nf_conntrack_standalone_init(void) 511 512 return register_pernet_subsys(&nf_conntrack_net_ops); 513 473 static int nf_conntrack_net_init(struct net *net) 474 475 int ret; 476 477 ret = nf_conntrack_init(net); 478 if (ret 0) 479 goto out_init; 480 ret = nf_conntrack_standalone_init_proc(net); 481 if (ret ct.sysctl_checksum = 1; 484 net-ct.sysctl_log_invalid = 0; 485 ret = nf_conntrack_standalone_init_sysctl(net); 486 if (ret 0) 487 goto out_sysctl; 488 return 0; 489 447-：/net/netfilter/nf_conntrack_core.c#nf_conntrack_init1275 int nf_conntrack_init(struct net *net) 1276 1277 int ret; 1278 1279 if (net_eq(net, &init_net) 1280 ret = nf_conntrack_init_init_net(); 1281 if (ret 0) 1282 goto out_init_net; 1283 1284 ret = nf_conntrack_init_net(net); 1285 if (ret ct.count, 0); 1228 INIT_HLIST_NULLS_HEAD(&net-ct.unconfirmed, 0); 1229 net-ct.stat = alloc_percpu(struct ip_conntrack_stat); 1234 ret = nf_conntrack_ecache_init(net); 1237 net-ct.hash = nf_ct_alloc_hashtable(&nf_conntrack_htable_size, 1238 &net-ct.hash_vmalloc, 1); 1244 ret = nf_conntrack_expect_init(net); 1247 ret = nf_conntrack_acct_init(net); 1253 #ifdef CONFIG_NET_NS 1254 nf_conntrack_untracked.ct_net = &init_net; 1255 #endif 1256 atomic_set(&nf_conntrack_untracked.ct_general.use, 1); 1257 /* - and look it like as a confirmed connection */ 1258 set_bit(IPS_CONFIRMED_BIT, &nf_conntrack_untracked.status); 1259 1260 return 0;这个函数所做的主要工作就是初始化net中的net-ct中的成员。初始化hash表大小，初始化expect链表大小，初始化acct扩展模块。最后nf_conntrack_untracked.ct_net = &init_net，再设置它的status位为IPS_CONFIRMED_BIT;三conntrack的扩展/include/net/netfilter/nf_conntrack_extend.h66struct nf_ct_ext_type6768/* Destroys relationships (can be NULL). */69void (*destroy)(struct nf_conn *ct);70/* Called when realloacted (can be NULL).71 Contents has already been moved. */72void (*move)(void *new, void *old);7374enum nf_ct_ext_id id;7576unsigned int flags;7778/* Length and min alignment. */79u8 len;80u8 align;81/* initial size of nf_ct_ext. */82u8 alloc_size;83;目前conntrack扩展有acct,helper,nat三种。它存储在*nf_ct_ext_typesNF_CT_EXT_NUM全局数组中。以nf_conntrack_acct为例：/net/netfilter/nf_conntrack_acct.c初始化59static struct nf_ct_ext_type acct_extend _read_mostly = 60.len= sizeof(struct nf_conn_counterIP_CT_DIR_MAX),61.align= _alignof_(struct nf_conn_counterIP_CT_DIR_MAX), 获得对齐方式62.id= NF_CT_EXT_ACCT,63;struct nf_conn_counter为计数用：17struct nf_conn_counter 18u_int64_t packets;19u_int64_t bytes;20;.len= sizeof(struct nf_conn_counterIP_CT_DIR_MAX),代表为连接的两个方向都要分配，进行计数。1.扩展的注册 /net/netfilter/nf_conntrack_acct.c110int nf_conntrack_acct_init(struct net *net)111112int ret;113114net-ct.sysctl_acct = nf_ct_acct;115116if (net_eq(net, &init_net) 123ret = nf_ct_extend_register(&acct_extend);124if (ret 0) 125printk(KERN_ERR nf_conntrack_acct: Unable to register extensionn);126goto out_extend_register;127128129130ret = nf_conntrack_acct_init_sysctl(net);131if (ret id) 166ret = -EBUSY;167goto out;168169170/* This ensures that nf_ct_ext_create() can allocate enough area171 before updating alloc_size */172type-alloc_size = ALIGN(sizeof(struct nf_ct_ext), type-align)173 + type-len;174rcu_assign_pointer(nf_ct_ext_typestype-id, type);175update_alloc_size(type);176out:177mutex_unlock(&nf_ct_ext_type_mutex);178return ret;179第172行对nf_ct_ext的size进行对齐，再加上type-len再赋值给type-alloc_size.即它的大小为2*nf_conn_counter+1*nf_ct_ext;然后再将type保存到nf_ext_types数组中。对照可以发现，struct nf_ct_ext_type仅仅只是保存了struct nf_ct_ext所必要的一些信息而已。至于为何要对齐，是因为扩展不只是一种，它们的数组结构也不一样，对齐方式都不一样，所以得保存自己的对齐方式，再对齐。2.添加一个acct扩展 nf_ct_acct_ext_add：/include/net/netfilter/nf_conntrack_acct.h28static inline29struct nf_conn_counter *nf_ct_acct_ext_add(struct nf_conn *ct, gfp_t gfp)3031struct net *net = nf_ct_net(ct);32struct nf_conn_counter *acct;3334if (!net-ct.sysctl_acct)35return NULL;3637acct = nf_ct_ext_add(ct, NF_CT_EXT_ACCT, gfp);38if (!acct)39pr_debug(failed to add accounting extension area);404142return acct;43;/include/net/netfilter/nf_conntrack_extend.h#6161#define nf_ct_ext_add(ct, id, gfp) 62(id#_TYPE *)_nf_ct_ext_add(ct), (id), (gfp)/net/netfilter/nf_conntrack_extend.c函数会判断ct-ext是否为NULL，若为NULL则证明这是第一个，所以调用nf_ct_ext_create（），它分配一个扩展的size并且返回它的起始地址；对于扩展的管理是先来的放前面。所以nf_ct_ext:offsetid会是添加id对应的扩展之前nf_ct_ext的对齐后的连同以前添加的扩展数据在内的长度。45static void *46nf_ct_ext_create(struct nf_ct_ext *ext, enum nf_ct_ext_id id, gfp_t gfp)4748unsigned int off, len;49struct nf_ct_ext_type *t;5051rcu_read_lock();52t = rcu_dereference(nf_ct_ext_typesid);53BUG_ON(t = NULL);54off = ALIGN(sizeof(struct nf_ct_ext), t-align);55len = off + t-len;56rcu_read_unlock();5758*ext = kzalloc(t-alloc_size, gfp);59if (!*ext)60return NULL;6162INIT_RCU_HEAD(&(*ext)-rcu);63(*ext)-offsetid = off;64(*ext)-len = len;6566return (void *)(*ext) + off;673.查找一个扩展/include/net/netfilter/nf_conntrack_extend.h#nf_ct_ext_find38#define nf_ct_ext_find(ext, id)39(id#_TYPE *)_nf_ct_ext_find(ext), (id) 对于acct就强制转换成NF_CT_EXT_ACCT_TYPE又：14#define NF_CT_EXT_HELPER_TYPE struct nf_conn_help15#define NF_CT_EXT_NAT_TYPE struct nf_conn_nat16#define NF_CT_EXT_ACCT_TYPE struct nf_conn_counter 所以转换成了struct nf_conn_counter类型。31static inline void *_nf_ct_ext_find(const struct nf_conn *ct, u8 id)3233if (!nf_ct_ext_exist(ct, id)34return NULL;3536return (void *)ct-ext + ct-ext-offsetid;37根据offset取出值。四conntrack的具体实现以PF_INET为例：/net/ipv4/netfilter/nf_conntrack_l3proto_ipv4.c 373 static int _init nf_conntrack_l3proto_ipv4_init(void) 374 375 int ret = 0; 376 377 need_conntrack(); 378 nf_defrag_ipv4_enable(); 379 380 ret = nf_register_sockopt(&so_getorigdst); 386 ret = nf_conntrack_l4proto_register(&nf_conntrack_l4proto_tcp4); 392 ret = nf_conntrack_l4proto_register(&nf_conntrack_l4proto_udp4); 398 ret = nf_conntrack_l4proto_register(&nf_conntrack_l4proto_icmp); 404 ret = nf_conntrack_l3proto_register(&nf_conntrack_l3proto_ipv4); 410 ret = nf_register_hooks(ipv4_conntrack_ops, 411 ARRAY_SIZE(ipv4_conntrack_ops); 416 #if defined(CONFIG_PROC_FS) & defined(CONFIG_NF_CONNTRACK_PROC_COMPAT) 417 ret = nf_conntrack_ipv4_compat_init(); 418 if (ret l3proto = PF_MAX) 272 if (l4proto-to_nlattr & !l4proto-nlattr_size) 273 | (l4proto-tuple_to_nlattr & !l4proto-nlattr_tuple_size) 274 return -EINVAL; 275 276 mutex_lock(&nf_ct_proto_mutex); 277 if (!nf_ct_protosl4proto-l3proto) 278 /* l3proto may be loaded latter. */ 279 struct nf_conntrack_l4proto *proto_array; 280 int i; 281 282 proto_array = kmalloc(MAX_NF_CT_PROTO * 283 sizeof(struct nf_conntrack_l4proto *), 284 GFP_KERNEL); 285 if (proto_array = NULL) 286 ret = -ENOMEM; 287 goto out_unlock; 288 289 290 for (i = 0; i l3proto = proto_array; 293 else if (nf_ct_protosl4proto-l3protol4proto-l4proto != 294 &nf_conntrack_l4proto_generic) 295 ret = -EBUSY; 296 goto out_unlock; 297 298 299 ret = nf_ct_l4proto_register_sysctl(l4proto); 300 if (ret nla_size = 0; 304 if (l4proto-nlattr_size) 305 l4proto-nla_size += l4proto-nlattr_size(); 306 if (l4proto-nlattr_tuple_size) 307 l4proto-nla_size += 3 * l4proto-nlattr_tuple_size(); 308 309 rcu_assign_pointer(nf_ct_protosl4proto-l3protol4proto-l4proto, 310 l4proto); 311 程序在277会检查nf_ct_protos数组中是否为NULL即是否已经有此三层协议注册过。如果没有，则分配相应的空间并且初始化成proto_arry，即把此三层协议对应的四层协议全部初始化成为nf_conntrack_l4proto_generic;如果不为NULL，并且对应的四层协议不为nf_conntrack_l4proto_generic说明出错。 最后，调用rcu_assign_pointer(nf_ct_protosl4proto-l3protol4proto-l4proto,l4proto);赋值。 三层协议的注册： 163int nf_conntrack_l3proto_register(struct nf_conntrack_l3proto *proto)164174if (nf_ct_l3protosproto-l3proto != &nf_conntrack_l3proto_generic) 179ret = nf_ct_l3proto_register_sysctl(proto);183if (proto-nlattr_tuple_size)184proto-nla_size = 3 * proto-nlattr_tuple_size();185186rcu_assign_pointer(nf_ct_l3protosproto-l3proto, proto);190return ret;191具体过程如四层协议的注册所说。进行检查，然后再赋值。2.钩子的注册： 钩子注册函数就不多说；看注册的钩子类型： 157static struct nf_hook_ops ipv4_conntrack_ops _read_mostly = 158159.hook= ipv4_conntrack_in,160.owner= THIS_MODULE,161.pf= PF_INET,162.hooknum= NF_INET_PRE_ROUTING,163.priority= NF_IP_PRI_CONNTRACK,164,165166.hook= ipv4_conntrack_local,167.owner= THIS_MODULE,168.pf= PF_INET,169.hooknum= NF_INET_LOCAL_OUT,170.priority= NF_IP_PRI_CONNTRACK,171,172173.hook= ipv4_confirm,174.owner= THIS_MODULE,175.pf= PF_INET,176.hooknum= NF_INET_POST_ROUTING,177.priority= NF_IP_PRI_CONNTRACK_CONFIRM,178,179180.hook= ipv4_confirm,181.owner= THIS_MODULE,182.pf= PF_INET,183.hooknum= NF_INET_LOCAL_IN,184.priority= NF_IP_PRI_CONNTRACK_CONFIRM,185,186;附上netfilter钩子点的图在4个钩子点分别注册了钩子函数。在NF_INET_PRE_ROUTING, NF_INET_LOCAL_OUT,钩子点处，即数据包进入和刚刚出去的点上的有非常高的优先级为NF_IP_PRI_CONNTRACK。数据包在NF_INET_PRE_ROUTING时候会进入钩子ipv4_conntrack_in，之后在通向更加高层次NF_INET_LOCAL_IN时会进入ipv4_confirm钩子。 作为server如果有一个新的连接在PRE_ROUTING收到数据包，则为ORIG,然后server回复给client,此数据包会经过LOCAL_OUT出去那么这时的包就称为REPLY,再之后双方的数据交换就ESTABLISH。即ORIG和REPLY代表第一个和第二个数据包。（1）ipv4_conntrack_in133static unsigned int ipv4_conntrack_in(unsigned int hooknum,134 struct sk_buff *skb,135 const struct net_device *in,136 const struct net_device *out,137 int (*okfn)(struct sk_buff *)138139return nf_conntrack_in(dev_net(in), PF_INET, hooknum, skb);140dev_net(in)取出net指针。由于是进入本机，所是参数为in;协议簇为PF_INET，hooknum为NF_INET_PRE_ROUTING；686nf_conntrack_in(struct net *net, u_int8_t pf, unsigned int hooknum,687struct sk_buff *skb)688689struct nf_conn *ct;690enum ip_conntrack_info ctinfo;691struct nf_conntrack_l3proto *l3proto;692struct nf_conntrack_l4proto *l4proto;693unsigned int dataoff;694u_int8_t protonum;695int set_reply = 0;696int ret;697704/* rcu_read_lock()ed by nf_hook_slow */705l3proto = _nf_ct_l3proto_find(pf);706ret = l3proto-get_l4proto(skb, skb_network_offset(skb),707 &dataoff, &protonum);715l4proto = _nf_ct_l4proto_find(pf, protonum);728729ct = resolve_normal_ct(net, skb, dataoff, pf, protonum,730 l3proto, l4proto, &set_reply, &ctinfo);745ret = l4proto-packet(ct, skb, dataoff, ctinfo, pf, hooknum);757758if (set_reply & !test_and_set_bit(IPS_SEEN_REPLY_BIT, &ct-status)759nf_conntrack_event_cache(IPCT_STATUS, ct);760761return ret;762-nf_conntrack_in程序会执行_nf_ct_l3proto_find(pf)从全局链表nf_conntrack_l3proto中取出 注册的l3协议；当然根据PF_INET取出的是nf_conntrack_l3proto_ipv4；于是执行ipv4_get_l4proto，此函数的主要功能就是从skb数据包中取出ip数据包中的 承载的协议类型，并且保存在protonum中。 然后根据找到的protonum和传入的pf从nf_ct_protospfprotonum来找到第四层协议。假设ip数据包承载的协议类型为IPPROTO_UDP;以后都以此假设；则取出的协议