lua 异步HTTPS并发请求库

项目使用skynet框架,这个框架主要用lua写逻辑,但缺乏对HTTPS支持,所以我利用一点时间写了lua模块,支持异步HTTPS请求,文章这里讲述HTTPS相关知识,如何接入openssl请求HTTPS数据,同时也分享了lua模块给大家参考。

HTTPS说明

HTTPS可以理解成 HTTP协议的安全版,协议还是HTTP协议,只是对传输过程的数据进行了加密处理,保证数据传输的安全。(默认端口是443)
HTTPS验证数据的过程如下:

skynet HTTPS支持

skynet只封装了HTTP的接口,没有对HTTPS做支持,所以要外接lua 库使用。

skynet支持HTTPS请求有两种方法:

优点 缺点
CURL 支持http/https/ftp等,接入较简单 并发支持差
OpenSSL 更底层,效率较高 接入复杂

其实,CURL是利用OpenSSL实现的,有网友封装了非阻塞版本的lua CURL库,可用于skynet处理HTTPS请求。链接猛击这里。

前面提到了CURL的缺点,CURL本身可做并发请求(libcurl multi),但做法却是将所有URL请求合到一起处理,需等全部URL数据处理完毕才返回数据。假设其中一个URL出现超时,那一起的其他URL都会受影响。

所以,文章推荐使用OpenSSL,这里先介绍C/C++如何处理HTTPS请求,然后再封装一个lua库,给大家演示下 skynet 如何请求HTTPS数据。

C/C++处理HTTPS请求

这里以一个例子,说明C/C++如何处理HTTPS请求。


#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <sys/socket.h>
#include <resolv.h>
#include <stdlib.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include "openssl/ssl.h"
#include "openssl/err.h"
 
#define MAXBUF 1024
 
int main(int argc, char **argv)
{
	int sockfd, len;
	char sendFN[1024];
	struct sockaddr_in dest;
	char buffer[MAXBUF + 1];
	SSL_CTX *ctx;
	SSL *ssl;
	char host_file[] = "";
	char  host_addr[] = "www.jd.com";
	char ip[] = "112.91.125.129";
	int port = "443";
 
	/* SSL 库初始化 */
	SSL_library_init();
	OpenSSL_add_all_algorithms();
	SSL_load_error_strings();
	ctx = SSL_CTX_new(SSLv23_client_method());
	if (ctx == NULL)
	{
		ERR_print_errors_fp(stdout);
		exit(1);
	}
	if ((sockfd = socket(AF_INET, SOCK_STREAM, 0)) < 0)
	{
		perror("Socket Create Fail!");
		exit(errno);
	}
	/* 建立 TCP 连接 */
	bzero(&dest, sizeof(dest));
	dest.sin_family = AF_INET;
	dest.sin_port = htons(atoi(port));
	if (inet_aton(ip, (struct in_addr *) &dest.sin_addr.s_addr) == 0)
	{
		perror("Socket Init Fail!");
		exit(errno);
	}
	printf("Socket Created\n");
 
	if (connect(sockfd, (struct sockaddr *) &dest, sizeof(dest)) != 0)
	{
		perror("Socket Connect Fail!");
		exit(errno);
	}
	printf("Socket Connected\n");
 
	/* 绑定 Socket 与 SSL */
	ssl = SSL_new(ctx);
	SSL_set_fd(ssl, sockfd);
	/* 建立 SSL 连接 */
	if (SSL_connect(ssl) == -1)
		ERR_print_errors_fp(stderr);
	else
		printf("SSL Connected with %s encryption\n", SSL_get_cipher(ssl));
 
 
	sprintf(sendFN, "GET /%s HTTP/1.1\r\nHost: %s\r\nConnection: Close\r\n\r\n", host_file, host_addr);
 
	/* SSL 发数据 */
	len = SSL_write(ssl, sendFN, strlen(sendFN));
	if (len < 0)
		printf("SSL Send failure! errno = %d, err_msg = %s\n", errno, strerror(errno));
	
	printf("SSL Send Done !\n\n");
 
	bzero(buffer, MAXBUF + 1);
	int nbytes;
	/* SSL 收数据 */
	while ((nbytes = SSL_read(ssl, buffer, 1)) == 1) {
		/* 打印收到的数据 */
		printf("%s", buffer);
	}
	
	printf("\n\nSSL Read Done !\n");
 
	/* 关闭连接 */
	SSL_shutdown(ssl);
	SSL_free(ssl);
	close(sockfd);
	SSL_CTX_free(ctx);
	return 0;
}

原理很简单,底层还是走了socket,只是TCP连接建立后,把连接交给SSL,让SSL收发数据。

Lua处理HTTPS请求

这里提供lua版本处理HTTPS请求,除了skynet,其他lua项目也可以使用。
以下是lua C模块的代码,保存为lua_httpsc.c


#include <stdlib.h>
#include <string.h>
#include <lua.h>
#include <lauxlib.h>
#include <time.h>
 
#include <netinet/in.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <unistd.h>
#include <errno.h>
#include <fcntl.h>
#include "openssl/ssl.h"
#include "openssl/err.h"
#include <poll.h>
 
 
#define CACHE_SIZE 0x1000
#define ERROR_FD -1
#define SEND_RETRY 10
 
static SSL_CTX *ctx = NULL;
 
typedef struct {
	int is_init;
} cutil_conf_t;
 
enum conn_st
{
	CONNECT_INIT = 1,
	CONNECT_PORT = 2,
	CONNECT_SSL = 3,
	CONNECT_DONE = 4
};
 
typedef struct {
	int fd;
	SSL* ssl;
	enum conn_st status;
} cutil_fd_t;
 
static cutil_conf_t* fetch_config(lua_State *L) {
	cutil_conf_t* cfg;
	cfg = lua_touserdata(L, lua_upvalueindex(1));
	if (!cfg)
		luaL_error(L, "httpsc: Unable to fetch cfg");
 
	return cfg;
}
 
static void close_fd_t(cutil_fd_t* fd_t) {
	if ( fd_t == NULL )
		return;
 
	SSL* ssl = fd_t->ssl;
	if ( ssl != NULL ) {
		SSL_shutdown(ssl);
		SSL_free(ssl);
	}
	
	int fd = fd_t->fd;
	if (fd != ERROR_FD)
		close(fd);
 
	free(fd_t);
}
 
static int try_connect_ssl(SSL* ssl) {
	int ret,err;
	ret = SSL_connect(ssl);
	err = SSL_get_error(ssl, ret);
	if (ret == 1) 
		return 0;
 
	if (err != SSL_ERROR_WANT_WRITE && err != SSL_ERROR_WANT_READ ) {
		return -1;
	}
	return 1;
}
 
static int lconnect(lua_State *L) {
	cutil_conf_t* cfg = fetch_config(L);
	if(!cfg->is_init)
	{
		luaL_error(L, "httpsc: Not inited");
		return 0;
	}
	
	const char * addr = luaL_checkstring(L, 1);
	int port = luaL_checkinteger(L, 2);
 
	cutil_fd_t* fd_t = (cutil_fd_t *)malloc(sizeof(cutil_fd_t));
	if ( fd_t == NULL )
		return luaL_error(L, "httpsc: Create fd %s %d failed", addr, port);
	fd_t->fd = ERROR_FD;
	fd_t->ssl = NULL;
	fd_t->status = CONNECT_INIT;
 
	
	int fd = socket(AF_INET, SOCK_STREAM, 0);
	struct sockaddr_in my_addr;
	fd_t->fd = fd;
 
	bzero(&my_addr, sizeof(my_addr));
	my_addr.sin_addr.s_addr = inet_addr(addr);
	my_addr.sin_family = AF_INET;
	my_addr.sin_port = htons(port);
 
	int ret;
	struct timeval timeo = {3, 0};
	socklen_t len = sizeof(timeo);
	ret = setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &timeo, len);
	if (ret) {
		close_fd_t(fd_t);
		return luaL_error(L, "httpsc: Setsockopt %s %d failed", addr, port);
	}
 
	int flag = fcntl(fd, F_GETFL, 0);
	fcntl(fd, F_SETFL, flag | O_NONBLOCK);
 
	ret = connect(fd, (struct sockaddr *)&my_addr, sizeof(struct sockaddr_in));
	if (ret != 0) {
		if (errno == EINPROGRESS) {
			fd_t->status = CONNECT_PORT;
		} else {
			close_fd_t(fd_t);
			return luaL_error(L, "httpsc: Connect %s %d failed", addr, port);
		}
 
	} else {
		fd_t->status = CONNECT_SSL;
		SSL *ssl = SSL_new(ctx);
		fd_t->ssl = ssl;
		SSL_set_fd(ssl, fd);
		ret = try_connect_ssl(ssl);
		if (ret == 0) {
			fd_t->status = CONNECT_DONE;
		} else if (ret == -1) {
			close_fd_t(fd_t);
			return luaL_error(L, "httpsc ssl_connect error, errno = %d", errno);
		}
	}
	
	lua_pushlightuserdata(L, fd_t);
 
	return 1;
}
 
static int lcheck_connect(lua_State *L) {
	cutil_fd_t* fd_t = (cutil_fd_t* ) lua_touserdata(L, 1);
	if ( fd_t == NULL )
		return luaL_error(L, "httpsc fd error");
 
	switch (fd_t->status) {
		case CONNECT_DONE:
			lua_pushboolean(L, 1);
			return 1;
		case CONNECT_PORT: 
			{
				struct pollfd fds;
				int ret, err;
				fds.fd = fd_t->fd;
				fds.events = POLLIN | POLLOUT;
				/* get status immediately */
				ret = poll(&fds, 1, 0);
 
				if (ret != -1) {
					socklen_t len = sizeof(int);
					ret = getsockopt(fd_t->fd, SOL_SOCKET, SO_ERROR, &err, &len);
					if (ret < 0) { close_fd_t(fd_t); return luaL_error(L, "httpsc getsockopt error, ret = %d", ret); } if (err == 0) { fd_t->status = CONNECT_SSL;
						SSL *ssl = SSL_new(ctx);
						fd_t->ssl = ssl;
						SSL_set_fd(ssl, fd_t->fd);
						ret = try_connect_ssl(ssl);
						if (ret == 0) {
							fd_t->status = CONNECT_DONE;
							lua_pushboolean(L, 1);
							return 1;
						} else if (ret == -1) {
							close_fd_t(fd_t);
							return luaL_error(L, "httpsc connect ssl error, errno = %d", errno);
						}
					} else {
						if (errno == EAGAIN || errno == EINTR || errno == EINPROGRESS ) {
							return 0;
						} else {
							close_fd_t(fd_t);
							return luaL_error(L, "httpsc connect sockopt error, errno = %d", errno);
						}
					}
				} else {
					close_fd_t(fd_t);
					return luaL_error(L, "httpsc connect poll error, ret = %d", ret);
				}
				return 0;
			}
		case CONNECT_SSL:
			{
				int ret = try_connect_ssl(fd_t->ssl);
				if (ret == 0) {
					fd_t->status = CONNECT_DONE;
					lua_pushboolean(L, 1);
					return 1;
				} else if (ret == -1) {
					close_fd_t(fd_t);
					return luaL_error(L, "httpsc connect ssl 2 error, errno = %d", errno);
				}
				return 0;
			}
		default:
			;
	}
 
	close_fd_t(fd_t);
	return luaL_error(L, "httpsc connect fator error");
}
 
static int lclose(lua_State *L) {
	cutil_fd_t* fd_t = (cutil_fd_t* ) lua_touserdata(L, 1);
	if ( fd_t == NULL )
		return luaL_error(L, "httpsc fd error");
	
	close_fd_t(fd_t);
 
	return 0;
}
 
static void block_send(lua_State *L, SSL *ssl, const char * buffer, int sz) {
	int retry = SEND_RETRY;
	while(sz > 0) {
		if (retry <= 0) {
			luaL_error(L, "httpsc: socket error: retry timeout");
			return;
		}
		retry -= 1;
		int r = SSL_write(ssl, buffer, sz);
		if (r < 0) { if (errno == EAGAIN || errno == EINTR) continue; luaL_error(L, "httpsc: socket error: %s", strerror(errno)); return; } buffer += r; sz -= r; } } static int lsend(lua_State *L) { cutil_conf_t* cfg = fetch_config(L); if(!cfg->is_init)
	{
		luaL_error(L, "httpsc: Not inited");
		return 0;
	}
	
	size_t sz = 0;
	cutil_fd_t* fd_t = (cutil_fd_t* ) lua_touserdata(L, 1);
	if ( fd_t == NULL )
		return luaL_error(L, "httpsc fd error");
	SSL* ssl = fd_t->ssl;
	const char * msg = luaL_checklstring(L, 2, &sz);
 
	block_send(L, ssl, msg, (int)sz);
 
	return 0;
}
 
 
static int lrecv(lua_State *L) {
	cutil_conf_t* cfg = fetch_config(L);
	if(!cfg->is_init)
	{
		luaL_error(L, "httpsc: Not inited");
		return 0;
	}
	cutil_fd_t* fd_t = (cutil_fd_t* ) lua_touserdata(L, 1);
	if ( fd_t == NULL )
		return luaL_error(L, "httpsc fd error");
	SSL* ssl = fd_t->ssl;
	int top = lua_gettop(L);
 
	char buffer[CACHE_SIZE];
	int size = CACHE_SIZE;
	if ( top > 1 && lua_isnumber(L, 2)) {
		int _size = lua_tointeger(L, 2);
		size = _size > size ? size : _size;
	}
 
	int r = SSL_read(ssl, buffer, size);
	// if (r == 0) {
	// 	lua_pushliteral(L, "");
	// 	/* close */
	// 	return 1;
	// }
	if (r <= 0) { if (errno == EAGAIN || errno == EINTR) { return 0; } return luaL_error(L, "httpsc: socket error: %s", strerror(errno)); } lua_pushlstring(L, buffer, r); return 1; } static int lusleep(lua_State *L) { int n = luaL_checknumber(L, 1); usleep(n); return 0; } /* GC, clean up the buf */ static int _gc(lua_State *L) { cutil_conf_t *cfg; cfg = lua_touserdata(L, 1); if (ctx != NULL){ SSL_CTX_free(ctx); ctx = NULL; } /* todo: auto gc */ cfg = NULL; return 0; } static void _create_config(lua_State *L) { cutil_conf_t *cfg; cfg = lua_newuserdata(L, sizeof(*cfg)); cfg->is_init = !!NULL;
	/* Create GC method to clean up buf */
	lua_newtable(L);
	lua_pushcfunction(L, _gc);
	lua_setfield(L, -2, "__gc");
	lua_setmetatable(L, -2);
 
	/* openssl init */
	if ( ctx == NULL) {
		SSL_library_init();
		OpenSSL_add_all_algorithms();
		SSL_load_error_strings();
		ctx = SSL_CTX_new(SSLv23_client_method());
		if (ctx == NULL)
		{
			ERR_print_errors_fp(stdout);
			luaL_error(L, "httpsc: Unable to init openssl");
			return;
		}
	}
 
	cfg->is_init = !NULL;
}
 
 
int luaopen_httpsc(lua_State *L)
{
	static const luaL_Reg funcs[] = {
		{ "connect", lconnect },
		{ "check_connect", lcheck_connect },
		{ "recv", lrecv },
		{ "send", lsend },
		{ "close", lclose },
		{ "usleep", lusleep },
		{NULL, NULL}
	};
 
	lua_newtable(L);
	_create_config(L);
	luaL_setfuncs(L, funcs, 1);
 
	return 1;
}

编译到项目执行后,lua调用的方法如下:


local httpsc = require "httpsc"
for k,v in pairs(httpsc ) do
	print(k,v)
end
 
local fd = httpsc.connect("163.177.151.109", 443)
print(fd)
 
while true do
	local ok = httpsc.check_connect(fd)
	if ok then break end
	httpsc.usleep(10000)
end
 
-- httpsc.usleep(10000)
httpsc.send(fd, "GET / HTTP/1.1\r\nAccept: */*\r\nHost: www.baidu.com\r\nConnection: Close\r\n\r\n")
 
httpsc.usleep(1000000)
 
local body = ""
while true do
	local r = httpsc.recv(fd)
	print(r)
	if r and #r>0 then
		body = body .. r
	else
		break
	end
end
 
print(body)
 
httpsc.close(fd)
 
print("ok!")

以上只是一个简单的例子,我同时也参考skynet clientsocket写了一个对HTTP协议支持友好的版本,代码托管在Git : https://github.com/chenweiqi/lua_httpsc ,欢迎交流。
这个lua代码目前有个不足,没有做openssl多线程支持,不支持同时多个线程加载和调用模块,这个以后的版本会做优化

OpenSSL动态库的编译

既然基于OpenSSL开发,就需要编译OpenSSL库来使用。
这里提供最近版本几个依赖库的编译方法:
openssl  – libssl.a / libcrypto.a

wget https://www.openssl.org/source/openssl-1.0.2k.tar.gz
tar -zxf openssl-1.0.2k.tar.gz
cd  openssl-1.0.2k
./config -fPIC enable-shared
make depend
make
make install
find -name "*.a"

zlib – zlib.a

wget http://zlib.net/zlib-1.2.11.tar.xz
tar -zxf zlib-1.2.11.tar.xz
cd zlib-1.2.11
export CFLAGS=" -fPIC"
./configure
make
find -name "*.a"

idn – libidn.a


wget http://ftp.gnu.org/gnu/libidn/libidn-1.33.tar.gz
tar -zxf libidn-1.33.tar.gz
cd libidn-1.33
export CFLAGS=" -fPIC"
./configure
make
find -name "*.a"

 

更新说明:
2017/4/15 优化socket创建连接过程,由阻塞改成非阻塞方式

发表评论

电子邮件地址不会被公开。 必填项已用*标注