Support automatic configuration of vlan and vlan ip, routing and the simplest policy routing

1. Automatic configuration of multi vlan vip(IPv4 and IPv6).
	1.1 And support automatic multi default route for per vlan, via different fib num.
	1.2 IPv6 not support set multi route fib now, just use RT_DEFAULT_FIB, And you can set multi fib use tool 'ff_route'.
	1.3 If vlan_flag is true, all port's addrs/vips will not to set, just create the iface.
2. Automatic configuration of simple policy routing.
	2.1 Only supports the simplest policy routing settings, like:
		`ff_ipfw -P 0 add 100 setfib 0 ip from 125.94.59.0/24 to any out`
		`from 125.94.59.0/24` need set addr is '125.94.59.0', netmask is '255.255.255.0'
	2.2 IPv6 does not support automatic configuration of policy routing. If ipv6 policy routing is required, you still need to use ff_ipfw to manually configure it.

config.ini

@@ -24,6 +24,11 @@ tso=0
 # HW vlan strip, default: enabled.
 vlan_strip=1
 
+# Set [vlanN]'s addrs like [portN] later
+# the format is same as port_list
+# Set vlan filter id, to enable L3/L4 RSS below vlan hdr is not enable after f-stack-1.22.
+vlan_filter=1,2,4-6
+
 # sleep when no pkts incomming
 # unit: microseconds
 idle_sleep=0
@@ -98,13 +103,19 @@ gateway=192.168.1.1
 #gateway6=ff::01
 
 # Multi virtual IPv4/IPv6 net addr, Optional parameters.
-#	`vip_ifname`: default `f-stack-x`
+#       `vip_ifname`: default `f-stack-x`
-#	`vip_addr`: Separated by semicolons, MAX number 64;
+#       `vip_addr`: Separated by semicolons, MAX number 64;
-#		    Only support netmask 255.255.255.255, broadcast x.x.x.255 now, hard code in `ff_veth_setvaddr`.
+#                   Only support netmask 255.255.255.255, broadcast x.x.x.255 now, hard code in `ff_veth_setvaddr`.
-#	`vip_addr6`: Separated by semicolons, MAX number 64.
+#       `ipfw_pr`: Set simplest policy routing,  Optional parameters.
-#	`vip_prefix_len`: All addr6 use the same prefix now, default 64.
+#                  Such as the cmd `ff_ipfw -P 0 add 100 setfib 0 ip from 192.168.0.0/24 to any out`
+#                  can set parameter`192.168.0.0 255.255.255.0`, cidr and netmask separated by space.
+#                  Multi cidr separated by semicolons.
+#                  IPv4 only now, and if you want set more complex policy routing, should use tool `ff_ipfw`.
+#       `vip_addr6`: Separated by semicolons, MAX number 64.
+#       `vip_prefix_len`: All addr6 use the same prefix now, default 64.
 #vip_ifname=lo0
-#vip_addr=192.168.1.3;192.168.1.4;192.168.1.5;192.168.1.6
+#vip_addr=192.168.0.3;192.168.0.4;192.168.0.5;192.168.0.6
+#ipfw_pr=192.168.0.0 255.255.255.0;192.168.10.0 255.255.255.0
 #vip_addr6=ff::03;ff::04;ff::05;ff::06;ff::07
 #vip_prefix_len=64
 
@@ -117,6 +128,55 @@ gateway=192.168.1.1
 # the format is same as port_list
 #slave_port_list=0,1
 
+# Vlan config section, Must set after all [portN]
+# NOTE1: Must enable dpdk.vlan_filter first, and match it.
+# NOTE2: If enable vlan config, all [PortN] config will be ignored!
+#[vlan1]
+#portid=0
+#addr=192.169.0.2
+#netmask=255.255.255.0
+#broadcast=192.169.0.255
+#gateway=192.169.0.1
+#
+#vip_addr=192.169.0.3;192.169.0.4;192.169.0.5;192.169.0.6
+#ipfw_pr=192.169.0.0 255.255.255.0;192.169.10.0 255.255.255.0
+#
+#[vlan2]
+#portid=0
+#addr=192.169.1.2
+#netmask=255.255.255.0
+#broadcast=192.169.1.255
+#gateway=192.169.1.1
+#
+#vip_addr=192.169.1.3;192.169.1.4;192.169.1.5;192.169.1.6
+#ipfw_pr=192.169.1.0 255.255.255.0;192.169.11.0 255.255.255.0
+#
+#[vlan4]
+#portid=0
+#addr=192.169.2.2
+#netmask=255.255.255.0
+#broadcast=192.169.2.255
+#gateway=192.169.2.1
+#
+#vip_addr=192.169.2.3;192.169.2.4;192.169.2.5;192.169.2.6
+#ipfw_pr=192.169.2.0 255.255.255.0;192.169.12.0 255.255.255.0
+#
+#[vlan5]
+#portid=0
+#addr=192.169.3.2
+#netmask=255.255.255.0
+#broadcast=192.169.3.255
+#gateway=192.169.3.1
+#
+#addr6=fe::32
+#prefix_len=64
+#gateway6=fe::31
+#
+#vip_addr=192.169.3.3;192.169.3.4;192.169.3.5;192.169.3.6
+#ipfw_pr=192.169.3.0 255.255.255.0;192.169.13.0 255.255.255.0
+#vip_addr6=fe::33;fe::34;fe::35;fe::36;fe::37
+#vip_prefix_len=64
+
 # Vdev config section
 # orrespond to dpdk.nb_vdev's index: vdev0, vdev1...
 #    iface : Shouldn't set always.

lib/Makefile

@@ -104,6 +104,7 @@ endif
 
 ifdef FF_IPFW
 HOST_CFLAGS+= -DFF_IPFW
+CFLAGS+= -DFF_IPFW
 endif
 
 ifdef FF_USE_PAGE_ARRAY

lib/ff_config.c

@@ -368,76 +368,170 @@ parse_port_slave_list(struct ff_port_cfg *cfg, const char *v_str)
 }
 
 static int
-vip_cfg_handler(struct ff_port_cfg *cur)
+parse_vlan_filter_list(struct ff_config *cfg, const char *v_str)
+{
+    cfg->dpdk.nb_vlan_filter = DPDK_MAX_VLAN_FILTER;
+    uint16_t *vlan_filter = cfg->dpdk.vlan_filter_id;
+    return __parse_config_list(vlan_filter, &cfg->dpdk.nb_vlan_filter, v_str);
+}
+
+static int
+vip_cfg_handler(struct ff_port_cfg *cur_port_cfg, struct ff_vlan_cfg *cur_vlan_cfg)
 {
     //vip cfg
-    int ret;
+    int ret, nb_vip;
-    char *vip_addr_array[VIP_MAX_NUM];
+    char *vip_addr_array[VIP_MAX_NUM], *vip_addr_str;
+    char **vip_addr_array_p;
 
-    ret = rte_strsplit(cur->vip_addr_str, strlen(cur->vip_addr_str), &vip_addr_array[0], VIP_MAX_NUM, ';');
+    if (cur_port_cfg) {
-    if (ret <= 0) {
+        vip_addr_str = cur_port_cfg->vip_addr_str;
-        fprintf(stdout, "vip_cfg_handler nb_vip is 0, not set vip_addr or set invalid vip_addr %s\n",
+    } else if (cur_vlan_cfg) {
-            cur->vip_addr_str);
+        vip_addr_str = cur_vlan_cfg->vip_addr_str;
+    } else {
+        fprintf(stdout, "vip_cfg_handler cur_port_cfg and cur_vlan_cfg both NULL, not set vip_addr\n");
         return 1;
     }
 
-    cur->nb_vip = ret;
+    ret = rte_strsplit(vip_addr_str, strlen(vip_addr_str), &vip_addr_array[0], VIP_MAX_NUM, ';');
+    if (ret <= 0) {
+        fprintf(stdout, "vip_cfg_handler nb_vip is 0, not set vip_addr or set invalid vip_addr %s\n",
+            vip_addr_str);
+        return 1;
+    }
 
-    cur->vip_addr_array = (char **)calloc(cur->nb_vip, sizeof(char *));
+    nb_vip = ret;
-    if (cur->vip_addr_array == NULL) {
+
+    vip_addr_array_p = (char **)calloc(nb_vip, sizeof(char *));
+    if (vip_addr_array_p == NULL) {
         fprintf(stderr, "vip_cfg_handler malloc failed\n");
         goto err;
     }
 
-    memcpy(cur->vip_addr_array, vip_addr_array, cur->nb_vip * sizeof(char *));
+    memcpy(vip_addr_array_p, vip_addr_array, nb_vip * sizeof(char *));
+
+    if (cur_port_cfg) {
+        cur_port_cfg->nb_vip = nb_vip;
+        cur_port_cfg->vip_addr_array = vip_addr_array_p;
+    } else if (cur_vlan_cfg) {
+        cur_vlan_cfg->nb_vip = nb_vip;
+        cur_vlan_cfg->vip_addr_array = vip_addr_array_p;
+    }
 
     return 1;
 
 err:
-    cur->nb_vip = 0;
-    if (cur->vip_addr_array) {
-        free(cur->vip_addr_array);
-        cur->vip_addr_array = NULL;
-    }
-
     return 0;
 }
 
 #ifdef INET6
 static int
-vip6_cfg_handler(struct ff_port_cfg *cur)
+vip6_cfg_handler(struct ff_port_cfg *cur_port_cfg, struct ff_vlan_cfg *cur_vlan_cfg)
 {
     //vip6 cfg
-    int ret;
+    int ret, nb_vip6;
-    char *vip_addr6_array[VIP_MAX_NUM];
+    char *vip_addr6_array[VIP_MAX_NUM], *vip_addr6_str;
+    char **vip_addr6_array_p;
 
-    ret = rte_strsplit(cur->vip_addr6_str, strlen(cur->vip_addr6_str),
+    if (cur_port_cfg) {
-                                    &vip_addr6_array[0], VIP_MAX_NUM, ';');
+        vip_addr6_str = cur_port_cfg->vip_addr6_str;
-    if (ret == 0) {
+    } else if (cur_vlan_cfg) {
-        fprintf(stdout, "vip6_cfg_handler nb_vip6 is 0, not set vip_addr6 or set invalid vip_addr6 %s\n",
+        vip_addr6_str = cur_vlan_cfg->vip_addr6_str;
-            cur->vip_addr6_str);
+    } else {
+        fprintf(stdout, "vip6_cfg_handler cur_port_cfg and cur_vlan_cfg both NULL, not set vip_addr\n");
         return 1;
     }
 
-    cur->nb_vip6 = ret;
+    ret = rte_strsplit(vip_addr6_str, strlen(vip_addr6_str),
-
+                                    &vip_addr6_array[0], VIP_MAX_NUM, ';');
-    cur->vip_addr6_array = (char **) calloc(cur->nb_vip6, sizeof(char *));
+    if (ret == 0) {
-    if (cur->vip_addr6_array == NULL) {
+        fprintf(stdout, "vip6_cfg_handler nb_vip6 is 0, not set vip_addr6 or set invalid vip_addr6 %s\n",
-        fprintf(stderr, "vip6_cfg_handler malloc failed\n");
+            vip_addr6_str);
-        goto fail;
+        return 1;
     }
 
-    memcpy(cur->vip_addr6_array, vip_addr6_array, cur->nb_vip6 * sizeof(char *));
+    nb_vip6 = ret;
+
+    vip_addr6_array_p = (char **) calloc(nb_vip6, sizeof(char *));
+    if (vip_addr6_array_p == NULL) {
+        fprintf(stderr, "vip6_cfg_handler malloc failed\n");
+        goto err;
+    }
+
+    memcpy(vip_addr6_array_p, vip_addr6_array, nb_vip6 * sizeof(char *));
+
+    if (cur_port_cfg) {
+        cur_port_cfg->nb_vip6 = nb_vip6;
+        cur_port_cfg->vip_addr6_array = vip_addr6_array_p;
+    } else if (cur_vlan_cfg) {
+        cur_vlan_cfg->nb_vip6 = nb_vip6;
+        cur_vlan_cfg->vip_addr6_array = vip_addr6_array_p;
+    }
 
     return 1;
 
-fail:
+err:
-    cur->nb_vip6 = 0;
+    return 0;
-    if (cur->vip_addr6_array) {
+}
-        free(cur->vip_addr6_array);
+#endif
-        cur->vip_addr6_array = NULL;
+
+#ifdef FF_IPFW
+static int
+ipfw_pr_cfg_handler(struct ff_port_cfg *cur_port_cfg, struct ff_vlan_cfg *cur_vlan_cfg)
+{
+    //vip cfg
+    int ret, nb_vip, i;
+    char *vip_addr_array[VIP_MAX_NUM], *vip_addr_mask_array[2], *vip_addr_str;
+    struct ff_ipfw_pr_cfg *vipfw_pr_cfg_p;
+
+    if (cur_port_cfg) {
+        vip_addr_str = cur_port_cfg->pr_addr_str;
+    } else if (cur_vlan_cfg) {
+        vip_addr_str = cur_vlan_cfg->pr_addr_str;
+    } else {
+        fprintf(stdout, "ipfw_pr_cfg_handlercur_port_cfg and cur_vlan_cfg both NULL, not set vip_addr\n");
+        return 1;
     }
 
+    ret = rte_strsplit(vip_addr_str, strlen(vip_addr_str), &vip_addr_array[0], VIP_MAX_NUM, ';');
+    if (ret <= 0) {
+        fprintf(stdout, "ipfw_pr_cfg_handler nb_vip is 0, not set vip_addr or set invalid vip_addr %s\n",
+            vip_addr_str);
+        return 1;
+    }
+
+    nb_vip = ret;
+
+    vipfw_pr_cfg_p = (struct ff_ipfw_pr_cfg *)calloc(nb_vip, sizeof(struct ff_ipfw_pr_cfg));
+    if (vipfw_pr_cfg_p == NULL) {
+        fprintf(stderr, "ipfw_pr_cfg_handler malloc failed\n");
+        goto err;
+    }
+
+    for (i = 0; i < nb_vip; i++) {
+        vip_addr_str = vip_addr_array[i];
+        ret = rte_strsplit(vip_addr_str, strlen(vip_addr_str), &vip_addr_mask_array[0], 2, ' ');
+        if (ret != 2) {
+            fprintf(stdout, "ipfw_pr_cfg_handler addr and netmask format error %s\n",
+                vip_addr_str);
+            free(vipfw_pr_cfg_p);
+            return 1;
+        };
+
+        vipfw_pr_cfg_p[i].addr = vip_addr_mask_array[0];
+        vipfw_pr_cfg_p[i].netmask = vip_addr_mask_array[1];
+    }
+
+    if (cur_port_cfg) {
+        cur_port_cfg->nb_pr = nb_vip;
+        cur_port_cfg->pr_cfg = vipfw_pr_cfg_p;
+    } else if (cur_vlan_cfg) {
+        cur_vlan_cfg->nb_pr = nb_vip;
+        cur_vlan_cfg->pr_cfg = vipfw_pr_cfg_p;
+    }
+
+    return 1;
+
+err:
     return 0;
 }
 #endif
@@ -507,12 +601,21 @@ port_cfg_handler(struct ff_config *cfg, const char *section,
     } else if (strcmp(name, "vip_addr") == 0) {
         cur->vip_addr_str = strdup(value);
         if (cur->vip_addr_str) {
-            return vip_cfg_handler(cur);
+            return vip_cfg_handler(cur, NULL);
         }
     } else if (strcmp(name, "vip_ifname") == 0) {
         cur->vip_ifname = strdup(value);
     }
 
+#ifdef FF_IPFW
+    else if (strcmp(name, "ipfw_pr") == 0) {
+        cur->pr_addr_str = strdup(value);
+        if (cur->pr_addr_str) {
+            return ipfw_pr_cfg_handler(cur, NULL);
+        }
+    }
+#endif
+
 #ifdef INET6
     else if (0 == strcmp(name, "addr6")) {
         cur->addr6_str = strdup(value);
@@ -523,7 +626,112 @@ port_cfg_handler(struct ff_config *cfg, const char *section,
     } else if (strcmp(name, "vip_addr6") == 0) {
         cur->vip_addr6_str = strdup(value);
         if (cur->vip_addr6_str) {
-            return vip6_cfg_handler(cur);
+            return vip6_cfg_handler(cur, NULL);
+        }
+    } else if (0 == strcmp(name, "vip_prefix_len")) {
+        cur->vip_prefix_len = atoi(value);
+    }
+#endif
+
+    return 1;
+}
+
+static int
+vlan_cfg_handler(struct ff_config *cfg, const char *section,
+    const char *name, const char *value) {
+    int vlanid, vlan_index, portid;
+    int ret;
+
+    if (cfg->dpdk.nb_vlan_filter == 0) {
+        fprintf(stderr, "vlan_cfg_handler: must config dpdk.vlan_filter first\n");
+        return 0;
+    }
+
+    if (cfg->dpdk.vlan_cfgs == NULL) {
+        struct ff_vlan_cfg *vc = calloc(DPDK_MAX_VLAN_FILTER, sizeof(struct ff_vlan_cfg));
+        if (vc == NULL) {
+            fprintf(stderr, "vlan_cfg_handler malloc failed\n");
+            return 0;
+        }
+        cfg->dpdk.vlan_cfgs = vc;
+    }
+
+    ret = sscanf(section, "vlan%d", &vlanid);
+    if (ret != 1) {
+        fprintf(stderr, "vlan_cfg_handler section[%s] error\n", section);
+        return 0;
+    }
+
+    /* just return true if vlanid not in vlan_filter */
+    for (vlan_index = 0; vlan_index < cfg->dpdk.nb_vlan_filter; vlan_index ++) {
+        if (vlanid == cfg->dpdk.vlan_filter_id[vlan_index]) {
+            break;
+        }
+
+        if (vlan_index >= cfg->dpdk.nb_vlan_filter) {
+            fprintf(stderr, "vlan_cfg_handler section[%s] mot match vlan filter, ignore it\n", section);
+            return 1;
+        }
+    }
+
+    struct ff_vlan_cfg *cur = &cfg->dpdk.vlan_cfgs[vlan_index];
+    if (cur->name == NULL) {
+        cur->name = strdup(section);
+        cur->vlan_id = vlanid;
+        cur->vlan_idx = vlan_index;
+    }
+
+    /* vlan not need `if_name`, should use [portN]'s `if_name` */
+    /*if (strcmp(name, "if_name") == 0) {
+        cur->ifname = strdup(value);
+    } else */
+    if (strcmp(name, "portid") == 0) {
+        portid = atoi(value);
+        if (portid > cfg->dpdk.max_portid) {
+            fprintf(stderr, "vlan_cfg_handler portid %d bigger than max port id\n", portid);
+            return 1;
+        }
+        struct ff_port_cfg *pc = &cfg->dpdk.port_cfgs[portid];
+        cur->port_id = portid;
+        pc->vlan_cfgs[pc->nb_vlan] = cur;
+        pc->nb_vlan++;
+    } else if (strcmp(name, "addr") == 0) {
+        cur->addr = strdup(value);
+    } else if (strcmp(name, "netmask") == 0) {
+        cur->netmask = strdup(value);
+    } else if (strcmp(name, "broadcast") == 0) {
+        cur->broadcast = strdup(value);
+    } else if (strcmp(name, "gateway") == 0) {
+        cur->gateway = strdup(value);
+    } else if (strcmp(name, "vip_addr") == 0) {
+        cur->vip_addr_str = strdup(value);
+        if (cur->vip_addr_str) {
+            return vip_cfg_handler(NULL, cur);
+        }
+    /*} else if (strcmp(name, "vip_ifname") == 0) {
+        cur->vip_ifname = strdup(value);*/
+    }
+
+#ifdef FF_IPFW
+    else if (strcmp(name, "ipfw_pr") == 0) {
+        cur->pr_addr_str = strdup(value);
+        if (cur->pr_addr_str) {
+            return ipfw_pr_cfg_handler(NULL, cur);
+        }
+    }
+#endif
+
+#ifdef INET6
+    else if (0 == strcmp(name, "addr6")) {
+        cur->addr6_str = strdup(value);
+    } else if (0 == strcmp(name, "prefix_len")) {
+        cur->prefix_len = atoi(value);
+    } else if (0 == strcmp(name, "gateway6")) {
+        cur->gateway6_str = strdup(value);
+    } else if (strcmp(name, "vip_addr6") == 0) {
+        cur->vip_addr6_str = strdup(value);
+        if (cur->vip_addr6_str) {
+            return vip6_cfg_handler(NULL, cur);
         }
     } else if (0 == strcmp(name, "vip_prefix_len")) {
         cur->vip_prefix_len = atoi(value);
@@ -689,6 +897,8 @@ ini_parse_handler(void* user, const char* section, const char* name,
         pconfig->dpdk.tx_csum_offoad_skip = atoi(value);
     } else if (MATCH("dpdk", "vlan_strip")) {
         pconfig->dpdk.vlan_strip = atoi(value);
+    } else if (MATCH("dpdk", "vlan_filter")) {
+        return parse_vlan_filter_list(pconfig, value);
     } else if (MATCH("dpdk", "idle_sleep")) {
         pconfig->dpdk.idle_sleep = atoi(value);
     } else if (MATCH("dpdk", "pkt_tx_delay")) {
@@ -723,6 +933,8 @@ ini_parse_handler(void* user, const char* section, const char* name,
         return freebsd_conf_handler(pconfig, "sysctl", name, value);
     } else if (strncmp(section, "port", 4) == 0) {
         return port_cfg_handler(pconfig, section, name, value);
+    } else if (strncmp(section, "vlan", 4) == 0) {
+        return vlan_cfg_handler(pconfig, section, name, value);
     } else if (strncmp(section, "vdev", 4) == 0) {
         return vdev_cfg_handler(pconfig, section, name, value);
     } else if (strncmp(section, "bond", 4) == 0) {

lib/ff_config.h

@@ -35,6 +35,7 @@ extern "C" {
 #define DPDK_CONFIG_NUM 16
 #define DPDK_CONFIG_MAXLEN 256
 #define DPDK_MAX_LCORE 128
+#define DPDK_MAX_VLAN_FILTER 128
 #define PCAP_SNAP_MINLEN 94
 #define PCAP_SAVE_MINLEN (2<<22)
 
@@ -58,12 +59,23 @@ struct ff_hw_features {
     uint8_t tx_tso;
 };
 
-struct ff_port_cfg {
+#ifdef FF_IPFW
+struct ff_ipfw_pr_cfg {
+    //uint32_t rule_num;
+    //uint32_t fib_num; /* Use portN or vlanN's idx * 100 */
+    char *addr;
+    char *netmask;
+};
+#endif
+
+struct ff_vlan_cfg {
     char *name;
     char *ifname;
+    /* global vlan idx, also use for route table's fib num */
+    int vlan_idx;
+    uint16_t vlan_id;
     uint16_t port_id;
-    uint8_t mac[6];
+
-    struct ff_hw_features hw_features;
     char *addr;
     char *netmask;
     char *broadcast;
@@ -74,6 +86,49 @@ struct ff_port_cfg {
     char **vip_addr_array;
     uint32_t nb_vip;
 
+    /* simple policy routing, only need rule num(100/200/300/400), ip/mask,fib num(0/1/2/3/4)  */
+    char *pr_addr_str;
+
+#ifdef FF_IPFW
+    struct ff_ipfw_pr_cfg *pr_cfg;
+    uint32_t nb_pr;
+#endif
+
+#ifdef INET6
+    char *addr6_str;
+    char *gateway6_str;
+    uint8_t prefix_len;
+
+    char *vip_addr6_str;
+    char **vip_addr6_array;
+    uint32_t nb_vip6;
+    uint8_t vip_prefix_len;
+#endif
+};
+
+struct ff_port_cfg {
+    char *name;
+    char *ifname;
+    uint16_t port_id;
+    uint8_t mac[6];
+    struct ff_hw_features hw_features;
+
+    char *addr;
+    char *netmask;
+    char *broadcast;
+    char *gateway;
+
+    char *vip_ifname;
+    char *vip_addr_str;
+    char **vip_addr_array;
+    uint32_t nb_vip;
+
+#ifdef FF_IPFW
+    char *pr_addr_str;
+    struct ff_ipfw_pr_cfg *pr_cfg;
+    uint32_t nb_pr;
+#endif
+
 #ifdef INET6
     char *addr6_str;
     char *gateway6_str;
@@ -89,6 +144,9 @@ struct ff_port_cfg {
     int nb_slaves;
     uint16_t lcore_list[DPDK_MAX_LCORE];
     uint16_t *slave_portid_list;
+
+    int nb_vlan;
+    struct ff_vlan_cfg *vlan_cfgs[DPDK_MAX_VLAN_FILTER];
 };
 
 struct ff_vdev_cfg {
@@ -154,6 +212,8 @@ struct ff_config {
         int tso;
         int tx_csum_offoad_skip;
         int vlan_strip;
+        int nb_vlan_filter;
+        uint16_t vlan_filter_id[DPDK_MAX_VLAN_FILTER];
         int symmetric_rss;
 
         /* sleep x microseconds when no pkts incomming */
@@ -173,6 +233,7 @@ struct ff_config {
         uint16_t log_level;
         // MAP(portid => struct ff_port_cfg*)
         struct ff_port_cfg *port_cfgs;
+        struct ff_vlan_cfg *vlan_cfgs;
         struct ff_vdev_cfg *vdev_cfgs;
         struct ff_bond_cfg *bond_cfgs;
     } dpdk;

lib/ff_veth.c

@@ -50,10 +50,14 @@
 #include <net/if_dl.h>
 #include <net/route.h>
 #include <net/route/route_ctl.h>
+#include <net/if_clone.h>
 
 #include <netinet/in.h>
 #include <netinet/in_var.h>
 #include <netinet6/nd6.h>
+#ifdef FF_IPFW
+#include <netinet/ip_fw.h>
+#endif
 
 #include <machine/atomic.h>
 
@@ -144,6 +148,63 @@ ff_veth_config(struct ff_veth_softc *sc, struct ff_port_cfg *cfg)
     return 0;
 }
 
+static int
+ff_veth_vlan_config(struct ff_veth_softc *sc, struct ff_vlan_cfg *cfg)
+{
+    int i, j;
+
+    inet_pton(AF_INET, cfg->addr, &sc->ip);
+    inet_pton(AF_INET, cfg->netmask, &sc->netmask);
+    inet_pton(AF_INET, cfg->broadcast, &sc->broadcast);
+    inet_pton(AF_INET, cfg->gateway, &sc->gateway);
+
+    if (cfg->nb_vip) {
+        for (i = 0, j = 0; i < cfg->nb_vip; ++i) {
+            if (inet_pton(AF_INET, cfg->vip_addr_array[i], &sc->vip[j])) {
+                j++;
+            } else {
+                printf("ff_veth_vlan_config inet_pton vip %s failed.\n", cfg->vip_addr_array[i]);
+            }
+        }
+
+        sc->nb_vip = j;
+    }
+
+#ifdef INET6
+    if (cfg->addr6_str) {
+        inet_pton(AF_INET6_LINUX, cfg->addr6_str, &sc->ip6);
+        printf("%s: Addr6: %s\n", sc->host_ifname, cfg->addr6_str);
+
+        if (cfg->gateway6_str) {
+            inet_pton(AF_INET6_LINUX, cfg->gateway6_str, &sc->gateway6);
+            printf("%s: Gateway6: %s\n", sc->host_ifname, cfg->gateway6_str);
+        } else {
+            printf("%s: No gateway6 config found.\n", sc->host_ifname);
+        }
+
+        sc->prefix_length = cfg->prefix_len == 0 ? 64 : cfg->prefix_len;
+    } else {
+        printf("%s: No addr6 config found.\n", sc->host_ifname);
+    }
+
+    if (cfg->nb_vip6) {
+        for (i = 0, j = 0; i < cfg->nb_vip6; ++i) {
+            if (inet_pton(AF_INET6_LINUX, cfg->vip_addr6_array[i], &sc->vip6[j])) {
+                j++;
+            } else {
+                printf("ff_veth_vlan_config inet_pton vip6 %s failed.\n", cfg->vip_addr6_array[i]);
+            }
+        }
+
+        sc->nb_vip6 = j;
+        sc->vip_prefix_length = cfg->vip_prefix_len == 0 ? 64 : cfg->vip_prefix_len;
+    }
+#endif /* INET6 */
+
+    return 0;
+}
+
+
 static void
 ff_veth_init(void *arg)
 {
@@ -152,6 +213,7 @@ ff_veth_init(void *arg)
 
     ifp->if_drv_flags |= IFF_DRV_RUNNING;
     ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
+    ifp->if_flags |= IFF_UP;
 }
 
 static void
@@ -166,6 +228,7 @@ ff_veth_stop(struct ff_veth_softc *sc)
     struct ifnet *ifp = sc->ifp;
 
     ifp->if_drv_flags &= ~(IFF_DRV_RUNNING|IFF_DRV_OACTIVE);
+    ifp->if_flags &= ~IFF_UP;
 }
 
 static int
@@ -376,11 +439,16 @@ ff_veth_qflush(struct ifnet *ifp)
 }
 
 static int
-ff_veth_setaddr(struct ff_veth_softc *sc)
+ff_veth_setaddr(struct ff_veth_softc *sc, const char *if_name)
 {
     struct in_aliasreq req;
     bzero(&req, sizeof req);
-    strcpy(req.ifra_name, sc->ifp->if_dname);
+
+    if (if_name) {
+        strcpy(req.ifra_name, if_name);
+    } else {
+        strcpy(req.ifra_name, sc->ifp->if_dname);
+    }
 
     struct sockaddr_in sa;
     bzero(&sa, sizeof(sa));
@@ -405,7 +473,7 @@ ff_veth_setaddr(struct ff_veth_softc *sc)
 }
 
 static int
-ff_veth_set_gateway(struct ff_veth_softc *sc)
+ff_veth_set_gateway(struct ff_veth_softc *sc, uint32_t fib_num)
 {
     struct rt_addrinfo info;
     struct rib_cmd_info rci;
@@ -434,16 +502,18 @@ ff_veth_set_gateway(struct ff_veth_softc *sc)
     nm.sin_addr.s_addr = 0;
     info.rti_info[RTAX_NETMASK] = (struct sockaddr *)&nm;
 
-    return rib_action(RT_DEFAULT_FIB, RTM_ADD, &info, &rci);
+    return rib_action(fib_num, RTM_ADD, &info, &rci);
 }
 
 static int
-ff_veth_setvaddr(struct ff_veth_softc *sc, struct ff_port_cfg *cfg)
+ff_veth_setvaddr(struct ff_veth_softc *sc, struct ff_port_cfg *cfg, const char *if_name)
 {
     struct in_aliasreq req;
     bzero(&req, sizeof req);
 
-    if (cfg->vip_ifname) {
+    if (if_name) {
+        strlcpy(req.ifra_name, if_name, IFNAMSIZ);
+    } else if (cfg->vip_ifname) {
         strlcpy(req.ifra_name, cfg->vip_ifname, IFNAMSIZ);
     } else {
         strlcpy(req.ifra_name, sc->ifp->if_dname, IFNAMSIZ);
@@ -483,13 +553,172 @@ done:
     return ret;
 }
 
+#ifdef FF_IPFW
+/*
+ * Only supports the simplest policy routing settings, like:
+ * `ff_ipfw -P 0 add 100 setfib 0 ip from 125.94.59.0/24 to any out`
+ * `from 125.94.59.0/24` need set addr is '125.94.59.0', netmask is '255.255.255.0'
+ *
+ * More policy routing settings should use tool `ff_ipfw`.
+ */
+static int
+ff_ipfw_add_simple_v4(struct ff_port_cfg *port_cfg, struct ff_vlan_cfg *vlan_cfg, uint32_t fib_num)
+{
+    struct ff_ipfw_pr_cfg *pr_cfg;
+    uint32_t nb_pr;
+    int ret = -1, fd, i;
+    int level = IPPROTO_IP, optname;
+    uint32_t rulebuf[1024];
+    socklen_t sz;
+    ip_fw3_opheader *op3;
+    ipfw_obj_ctlv *ctlv;
+    struct ip_fw_rule *rule;
+    ipfw_insn *cmd;
+    ipfw_insn *action;
+    ipfw_insn_ip *cmd_ip;
+    int rule_len, ctlv_count = 1, cmd_count = 5, act_ofs = 4, cmd_ip_count = 3, action_count = 1;
+    uint32_t rule_num;
+
+    if (port_cfg) {
+        pr_cfg = port_cfg->pr_cfg;
+        nb_pr = port_cfg->nb_pr;
+    } else if (vlan_cfg) {
+        pr_cfg = vlan_cfg->pr_cfg;
+        nb_pr = vlan_cfg->nb_pr;
+    } else {
+        return -1;
+    }
+
+    fd = ff_socket(AF_INET, SOCK_RAW, IPPROTO_RAW);
+    if (fd < 0) {
+        printf("ff_set_ipfw ff_socket error\n");
+        goto done;
+    }
+
+    memset(rulebuf, 0, sizeof(rulebuf));
+    /* Because `struct ip_fw_rule` has 1 cmd(`ipfw_insn`), so should (cmd_count - 1), and then routeup2 8 bytes */
+    rule_len = sizeof(struct ip_fw_rule) + (cmd_count - 1) * sizeof(ipfw_insn);
+    rule_len = roundup2(rule_len, sizeof(uint64_t));
+    sz = sizeof(ip_fw3_opheader) + sizeof(ipfw_obj_ctlv) + rule_len;
+
+    op3 = (ip_fw3_opheader *)rulebuf;
+    op3->opcode = IP_FW_XADD;
+
+    /*
+     * {head = {type = 3, flags = 0, length = 56}, count = 1, objsize = 0, version = 0 '\000', flags = 0 '\000'}
+     * type:IPFW_TLV_RULE_LIST(3), length:ctlv header length + rule total length
+     */
+    ctlv = (ipfw_obj_ctlv *)(op3 + 1);
+    ctlv->head.type = IPFW_TLV_RULE_LIST;
+    ctlv->head.length = sizeof(ipfw_obj_ctlv) + rule_len;
+    ctlv->count = ctlv_count;
+    //rule->rulenum = rule_num;
+    rule = (struct ip_fw_rule *)(ctlv + 1);
+
+
+    /* Need rule num(100/200/300/400), ip/mask,fib num(0/1/2/3/4) */
+    /* $124 = {act_ofs = 4, cmd_len = 5, spare = 0, set = 0 '\000', flags = 0 '\000', rulenum = 200, id = 0, cmd = {{opcode = 2 '\002', len = 3 '\003', arg1 = 0}}}
+     * act_ofs is 4 means jump four cmd, it's point to the first action's index,
+     * cmd_len means total nums of cmd and action, per 4 bytes(sizeof(ipfw_insn)).
+     */
+    rule->act_ofs = act_ofs;
+    rule->cmd_len = cmd_count;
+
+    /*
+     * cmd: 3 + 1
+     * src IP:
+        cmd[0] {opcode = 2 '\002', len = 3 '\003', arg1 = 0}
+            opcode is 2(O_IP_SRC_MASK),
+            len = 3 means three 4 bytes,
+
+        [0] means cmd itself.
+            (gdb) p *(ipfw_insn_ip *)cmd
+            $61 = {o = {opcode = 2 '\002', len = 3 '\003', arg1 = 0}, addr = {s_addr = 373155}, mask = {s_addr = 16777215}}
+
+        [1] means `IP & mask`'s' value.
+        [2] means netmask, sucn as /24 is 0xffffff00
+            (gdb) x/12x cmd
+            0xe74860 <cmdbuf.7188>: 0x02    0x03    0x00    0x00    0xa3    0xb1    0x05    0x00
+            0xe74868 <cmdbuf.7188+8>:       0xff    0xff    0xff    0x00
+     *
+     * out:
+        cmd[3]
+            i = 40 // TOK_OUT
+            cmd->len ^= F_NOT;
+                cmd->len = 128
+            {opcode = 15 '\017', len = 129 '\201', arg1 = 0}
+                opcode is 15(O_IN none),
+                len=129 & 3f = 1, just jump one 4 bytes.
+                    Although len is 129, but just have one header, no actual data.
+     */
+    cmd_ip = (ipfw_insn_ip *)((ipfw_insn *)(rule + 1) - 1);
+    cmd_ip->o.opcode = O_IP_SRC_MASK;
+    cmd_ip->o.len = cmd_ip_count;
+    //inet_pton(AF_INET, addr, (void *)&(cmd_ip->addr));
+    //inet_pton(AF_INET, netmask, (void *)&(cmd_ip->mask));
+
+    cmd = (ipfw_insn *)(cmd_ip + 1);
+    cmd->opcode = O_IN;
+    cmd->len ^= F_NOT;
+    cmd->len =  ((cmd->len | 0) & (F_NOT | F_OR)) | 1; // flags = 0
+
+    /*
+     * action:1
+     * setfib 0: // action[0]
+        i = TOK_SETFIB	// 120
+        action->len = 1;
+        action->opcode = O_SETFIB; // 82
+        action->arg1 = strtoul(*av, NULL, 10); // fib num: 0、1、2、3
+        action->arg1 |= 0x8000;  // 32768 + fib num
+
+        (gdb) p *action
+        $34 = {opcode = 82 'R', len = 1 '\001', arg1 = 32769}
+
+        (gdb) x/4b actbuf
+        0xe74460 <actbuf.7187>: 0x52    0x01    0x01    0x80
+     */
+    action = (ipfw_insn *)(cmd + 1);
+    action->opcode = O_SETFIB;
+    action->len = action_count;
+    //action->arg1 = fib_num;
+    //action->arg1 |= 0x8000;
+
+    for (i = 0; i < nb_pr; i++) {
+        rule_num = (fib_num + 1) * 10;
+        rule->rulenum = rule_num;
+        inet_pton(AF_INET, pr_cfg[i].addr, (void *)&(cmd_ip->addr));
+        inet_pton(AF_INET, pr_cfg[i].netmask, (void *)&(cmd_ip->mask));
+        action->arg1 = fib_num;
+        action->arg1 |= 0x8000;
+
+        ret = ff_getsockopt(fd, IPPROTO_IP, IP_FW3, (void *)op3, (socklen_t *)&sz);
+        if (ret < 0) {
+            printf("ff_set_ipfw kern_getsockopt error\n");
+            goto done;
+        }
+    }
+
+done:
+    if (fd > 0) {
+        ff_close(fd);
+    }
+
+    return ret;
+}
+#endif
+
 #ifdef INET6
 static int
-ff_veth_setaddr6(struct ff_veth_softc *sc)
+ff_veth_setaddr6(struct ff_veth_softc *sc, const char *if_name)
 {
     struct in6_aliasreq ifr6;
     bzero(&ifr6, sizeof(ifr6));
-    strcpy(ifr6.ifra_name, sc->ifp->if_dname);
+
+    if (if_name) {
+        strcpy(ifr6.ifra_name, if_name);
+    } else {
+        strcpy(ifr6.ifra_name, sc->ifp->if_dname);
+    }
 
     ifr6.ifra_addr.sin6_len = sizeof ifr6.ifra_addr;
     ifr6.ifra_addr.sin6_family = AF_INET6;
@@ -518,7 +747,7 @@ ff_veth_setaddr6(struct ff_veth_softc *sc)
 }
 
 static int
-ff_veth_set_gateway6(struct ff_veth_softc *sc)
+ff_veth_set_gateway6(struct ff_veth_softc *sc, uint32_t fib_num)
 {
     struct sockaddr_in6 gw, dst, nm;;
     struct rt_addrinfo info;
@@ -541,16 +770,18 @@ ff_veth_set_gateway6(struct ff_veth_softc *sc)
     info.rti_info[RTAX_DST] = (struct sockaddr *)&dst;
     info.rti_info[RTAX_NETMASK] = (struct sockaddr *)&nm;
 
-    return rib_action(RT_DEFAULT_FIB, RTM_ADD, &info, &rci);
+    return rib_action(fib_num, RTM_ADD, &info, &rci);
 }
 
 static int
-ff_veth_setvaddr6(struct ff_veth_softc *sc, struct ff_port_cfg *cfg)
+ff_veth_setvaddr6(struct ff_veth_softc *sc, struct ff_port_cfg *cfg, const char *if_name)
 {
     struct in6_aliasreq ifr6;
     bzero(&ifr6, sizeof(ifr6));
 
-    if (cfg->vip_ifname) {
+    if (if_name) {
+        strlcpy(ifr6.ifra_name, if_name, IFNAMSIZ);
+    } else if (cfg->vip_ifname) {
         strlcpy(ifr6.ifra_name, cfg->vip_ifname, IFNAMSIZ);
     } else {
         strlcpy(ifr6.ifra_name, sc->ifp->if_dname, IFNAMSIZ);
@@ -597,6 +828,8 @@ static int
 ff_veth_setup_interface(struct ff_veth_softc *sc, struct ff_port_cfg *cfg)
 {
     struct ifnet *ifp;
+    int ret;
+    uint32_t fib_num = RT_DEFAULT_FIB;
 
     ifp = sc->ifp = if_alloc(IFT_ETHER);
 
@@ -632,42 +865,135 @@ ff_veth_setup_interface(struct ff_veth_softc *sc, struct ff_port_cfg *cfg)
     if (sc->host_ctx == NULL) {
         printf("%s: Failed to register dpdk interface\n", sc->host_ifname);
         return -1;
+    } else {
+        printf("%s: Successed to register dpdk interface\n", sc->host_ifname);
     }
 
-    // Set ip
+    /* if vlan_flag is true, all port's addrs/vips will not to set, just create the iface */
-    int ret = ff_veth_setaddr(sc);
+    if (cfg->nb_vlan == 0) {
-    if (ret != 0) {
+        // Set IP
-        printf("ff_veth_setaddr failed\n");
+        ret = ff_veth_setaddr(sc, NULL);
-    }
+        if (ret != 0) {
-    ret = ff_veth_set_gateway(sc);
+            printf("%s: ff_veth_setaddr failed\n", sc->host_ifname);
-    if (ret != 0) {
+        }
-        printf("ff_veth_set_gateway failed\n");
-    }
 
-    if (sc->nb_vip) {
+        fib_num = cfg->port_id;
-        ret = ff_veth_setvaddr(sc, cfg);
+        ret = ff_veth_set_gateway(sc, fib_num);
-    }
+        if (ret != 0) {
+            printf("%s: ff_veth_set_gateway failed\n", sc->host_ifname);
+        }
+
+        if (sc->nb_vip) {
+            ret = ff_veth_setvaddr(sc, cfg, NULL);
+        }
+
+#ifdef FF_IPFW
+        if (cfg->nb_pr) {
+            ff_ipfw_add_simple_v4(cfg, NULL, fib_num);
+        }
+#endif
 
 #ifdef INET6
-    // Set IPv6
+        // Set IPv6
-    if (cfg->addr6_str) {
+        if (cfg->addr6_str) {
-        ret = ff_veth_setaddr6(sc);
+            ret = ff_veth_setaddr6(sc, NULL);
-        if (ret != 0) {
-            printf("ff_veth_setaddr6 failed\n");
-        }
-
-        if (cfg->gateway6_str) {
-            ret = ff_veth_set_gateway6(sc);
             if (ret != 0) {
-                printf("ff_veth_set_gateway6 failed\n");
+                printf("%s: ff_veth_setaddr6 failed\n", sc->host_ifname);
+            }
+
+            if (cfg->gateway6_str) {
+                /*
+                 * IPv6 not support set multi route fib now, just use RT_DEFAULT_FIB.
+                 * And you can set multi fib use tool 'ff_route' a
+                 */
+                fib_num = RT_DEFAULT_FIB;
+                ret = ff_veth_set_gateway6(sc, fib_num);
+                if (ret != 0) {
+                    printf("%s: ff_veth_set_gateway6 failed\n", sc->host_ifname);
+                }
             }
         }
-    }
 
-    if (sc->nb_vip6) {
+        if (sc->nb_vip6) {
-        ret = ff_veth_setvaddr6(sc, cfg);
+            ret = ff_veth_setvaddr6(sc, cfg, NULL);
-    }
+        }
 #endif /* INET6 */
+    }else {
+        char vlan_if_name[IFNAMSIZ];
+        int idx;
+        struct ff_vlan_cfg *vlan_cfg;
+        struct ff_veth_softc vlan_sc;
+
+        ff_veth_init(sc);
+
+        for (idx = 0; idx < cfg->nb_vlan; idx++) {
+            /* create vlan interface */
+            vlan_cfg = cfg->vlan_cfgs[idx];
+            snprintf(vlan_if_name, IFNAMSIZ, "%s.%d", sc->host_ifname, vlan_cfg->vlan_id);
+            ret = if_clone_create(vlan_if_name, IFNAMSIZ, NULL);
+            if (0 != ret) {
+                printf("%s: Failed to if_clone_create vlan interface\n", vlan_if_name);
+                return -1;
+            } else {
+                printf("%s: Successed to if_clone_create vlan interface\n", vlan_if_name);
+            }
+
+            // Set vlan IP
+            memset(&vlan_sc, 0, sizeof(vlan_sc));
+            ret = ff_veth_vlan_config(&vlan_sc, vlan_cfg);
+            if (0 != ret) {
+                printf("%s: Failed to ff_veth_vlan_config\n", vlan_if_name);
+                return -1;
+            }
+
+            ret = ff_veth_setaddr(&vlan_sc, vlan_if_name);
+            if (ret != 0) {
+                printf("%s: ff_veth_setaddr failed\n", vlan_if_name);
+            }
+
+            fib_num = vlan_cfg->vlan_idx;
+            ret = ff_veth_set_gateway(&vlan_sc, fib_num);
+            if (ret != 0) {
+                printf("%s: ff_veth_set_gateway of fib %u failed\n", vlan_if_name, fib_num);
+            }
+
+            if (vlan_sc.nb_vip) {
+                ret = ff_veth_setvaddr(&vlan_sc, cfg, vlan_if_name);
+            }
+
+#ifdef FF_IPFW
+            if (vlan_cfg->nb_pr) {
+                ff_ipfw_add_simple_v4(NULL, vlan_cfg, fib_num);
+            }
+#endif
+
+#ifdef INET6
+            // Set vlan IPv6
+            if (vlan_cfg->addr6_str) {
+                ret = ff_veth_setaddr6(&vlan_sc, vlan_if_name);
+                if (ret != 0) {
+                    printf("%s: ff_veth_setaddr6 failed\n", vlan_if_name);
+                }
+
+                if (vlan_cfg->gateway6_str) {
+                    /*
+                     * IPv6 not support set multi route fib now, just use RT_DEFAULT_FIB.
+                     * And you can set multi fib use tool 'ff_route' a
+                     */
+                    fib_num = RT_DEFAULT_FIB;
+                    ret = ff_veth_set_gateway6(&vlan_sc, fib_num);
+                    if (ret != 0) {
+                        printf("%s: ff_veth_set_gateway6 of fib %u failed\n", vlan_if_name, fib_num);
+                    }
+                }
+            }
+
+            if (vlan_sc.nb_vip6) {
+                ret = ff_veth_setvaddr6(&vlan_sc, cfg, vlan_if_name);
+            }
+#endif /* INET6 */
+        }
+    }
 
     return (0);
 }