PHP结合Curl实现资源共享
分类： php、后端 2156 0
假设一个场景，ui同学在设计一张海报，在各大素材网站寻找素材。当他下载素材时，发现需要vip, 也就是充值，于是他在这个素材网站充值了。当他下次设计海报，发现这个素材网站没有他所需要的素材。于是，他在其他的素材网站上找到了自己所需要的素材，但是同样也需要vip。
试想一下，能不能通过技术的手段，从中间做一层代理，提供一个统一的下载接口，可以让用户下载各个网站上的素材呢？当然各大素材的网站会员还是要办的，但是只办一次，来提供给很多用户来使用。
实现思路:
首先观察一下 “觅图网”素材的下载过程。
以这个为例：http://www.51yuansu.com/sc/lcvnlvrkig.html
点击“下载透明PNG”, 会发起一个http请求。
是否可以使用 curl 来模拟 request header, request body，
（由于浏览器跨域问题，所以使用 php 进行开发）
主要代码实现：
<?php
namespace app\index\controller;
use app\common\model\Attach;
use app\common\model\MemberLog;
use app\common\model\ParseUrl;
use app\common\model\WebSite;
use app\common\model\WebSiteCookie;
use think\Controller;
use think\Db;
class Index extends Controller
{
protected function initialize()
{
global $_G;
if ($_G['setting']['must_login'] && empty($_G['uid'])) {
return $this->redirect('index/account/login');
}
$this->view->site_list = WebSite::where('status', '>', 0)->select();
}
public function index()
{
global $_G;
if (!empty($_G['uid']) && $_G['setting']['parse_between_time'] > 0) {
$last = MemberLog::where([['uid', '=', $_G['member']['uid']], ['status', '=', 1], ['create_time', '>=', $this->request->time() - $_G['setting']['parse_between_time']]])->find();
$between_time = $_G['setting']['parse_between_time'] - ($this->request->time() - strtotime($last['create_time']));
if ($between_time > 0) {
$this->view->between_time = $between_time;
}
}
return $this->fetch();
}
public function parse($link = '', $debug = '', $verify = '')
{
global $_G;
if (empty($link)) {
return $this->error('请输入需要解析的网址');
}
if (empty($_G['member'])) {
return $this->error('请先登陆后再使用此功能');
}
if ($_G['member']['out_time'] > 0 && $_G['member']['out_time'] <= request()->time()) {
return $this->error('您的账户已过期，请联系管理员！');
}
if ($_G['member']['parse_max_times'] < 0) {
return $this->error('您的账户没有解析权限');
}
if ($_G['member']['parse_max_times'] > 0 && $_G['member']['parse_times'] >= $_G['member']['parse_max_times']) {
return $this->error('您的账户解析次数已达上限，请充值');
}
if ($_G['setting']['parse_between_time'] > 0 && $last = MemberLog::where([['uid', '=', $_G['member']['uid']], ['status', '=', 1], ['create_time', '>=', $this->request->time() - $_G['setting']['parse_between_time']]])->find()) {
$between_time = $_G['setting']['parse_between_time'] - ($this->request->time() - strtotime($last['create_time']));
return $this->error('操作太频繁啦，请' . $between_time . '秒再试！');
}
$site = '';
foreach ($this->view->site_list as $data) {
if (stripos($link, $data['url_regular']) !== false) {
$site = $data;
break;
}
}
if (empty($site) || $site['status'] != 1) {
return $this->error('暂不支持该网址的解析！');
}
if (empty($_G['member']['site_access'][$site['site_id']])) {
return $this->error('您没有该网站的解析权限，请联系管理员或充值');
}
$access = $_G['member']['site_access'][$site['site_id']];
if ($access['day'] < 0 || $access['all'] < 0) {
return $this->error('您没有该网站的解析权限，请联系管理员或充值');
}
if ($access['day'] > 0 && $access['day_used'] >= $access['day']) {
return $this->error('目标网站今日的解析次数已用完，试试其他网站吧');
}
if ($access['all'] > 0 && $access['max_used'] >= $access['all']) {
return $this->error('目标站解析次数已达上限，请联系客服充值');
}
$action = 'get_' . str_replace('.', '_', $site['url_regular']);
$ParseUrl = new ParseUrl($link, $site);
$cookie = $ParseUrl->cookie;
if (empty($cookie) || !method_exists($ParseUrl, $action)) {
return $this->error('暂不支持该网址的解析！');
}
if (config('app.app_debug') == true) {
$result = $ParseUrl->$action($verify, $access);
} else {
try {
$result = $ParseUrl->$action($verify, $access);
} catch (\Exception $e) {
return $this->error('解析失败，错误码：500');
}
}
if ($debug == 'nanbowan_debug') {
var_dump($result);
exit;
}
if ($result['code'] === 'verify') {
return json($result);
} else if ($result['code'] !== 1) {
return $this->error($result['msg']);
}
if (empty($result['has_attach'])) {
foreach ($result['msg'] as $button_name => $download_url) {
Attach::create([
'site_id' => $site['site_id'],
'request_url' => $link,
'site_code_type' => $result['site_code_type'],
'site_code' => $result['site_code'],
'response_url' => $download_url,
'button_name' => $button_name,
'queue_error' => '',
'cookie_id' => $cookie['cookie_id'],
]);
}
}
if (empty($result['use_times'])) {
$result['use_times'] = 1;
}
MemberLog::create([
'uid' => $_G['member']['uid'],
'site_id' => $site['site_id'],
'times' => $result['use_times'],
'status' => $result['use_times'],
'parse_url' => $link,
]);
$access['day_used'] = $access['day_used'] + $result['use_times'];
$access['max_used'] = $access['max_used'] + $result['use_times'];
$site_access = $_G['member']->site_access;
$site_access[$site['site_id']] = $access;
$_G['member']->site_access = $site_access;
$_G['member']->parse_times = $_G['member']->parse_times + $result['use_times'];
$_G['member']->save();
WebSiteCookie::where('cookie_id', '=', $cookie['cookie_id'])->update(['used_times' => Db::raw('used_times+' . $result['use_times'])]);
return $this->success('解析成功！', '', $result['msg']);
}
public function demo()
{
return $this->fetch();
}
public function jietu()
{
return $this->fetch();
}
public function price()
{
return $this->fetch();
}
public function update_log()
{
return $this->fetch();
}
}根据用户输入的域名来匹配是哪个运营商，来执行对应的方法。以这个为例：http://www.51yuansu.com/sc/lcvnlvrkig.html
public function get_51yuansu_com()
{
preg_match('/\/(\w+).html/', $this->link, $site_code);
if (empty($site_code['1'])) {
return ['code' => 0, 'msg' => '解析失败，网址输入错误或不支持该站点解析'];
}
$cache = $this->get_cache($site_code['1']);
if ($cache !== false) {
return $cache;
}
$header = [
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
'Host' => 'www.51yuansu.com',
];
$http_curl = new HttpCurl($this->link, 'GET', null, $header, true, $this->cookie['cookie_id']);
$html = $http_curl->send_request()->get_response_body();
$header['Accept'] = 'application/json, text/javascript, */*; q=0.01';
$header['Referer'] = $this->link;
$header['X-Requested-With'] = 'XMLHttpRequest';
$download = [];
if (preg_match('/data-id="' . $site_code['1'] . '".*?class="p-down-operate /', $html) > 0) {
$http_curl = new HttpCurl('http://www.51yuansu.com/index.php?m=ajax&a=down&id=' . $site_code['1'], 'GET', null, $header, true, $this->cookie['cookie_id']);
$response = $http_curl->send_request()->get_response_body();
$response = json_decode($response, true);
if (!empty($response['url'])) {
$download['PNG下载'] = $response['url'];
}
}
if (preg_match('/data-id="' . $site_code['1'] . '".*?class="p-down-operate-zip /', $html) > 0) {
$http_curl = new HttpCurl('http://www.51yuansu.com/index.php?m=ajax&a=downPsd&id=' . $site_code['1'], 'GET', null, $header, true, $this->cookie['cookie_id']);
$response = $http_curl->send_request()->get_response_body();
$response = json_decode($response, true);
if (!empty($response['url'])) {
$download['PSD下载'] = $response['url'];
}
}
if (preg_match('/data-id="' . $site_code['1'] . '".*?class="b-down-operate-zip /', $html) > 0) {
$http_curl = new HttpCurl('http://www.51yuansu.com/index.php?m=ajax&a=bdownPsd&id=' . $site_code['1'], 'GET', null, $header, true, $this->cookie['cookie_id']);
$response = $http_curl->send_request()->get_response_body();
$response = json_decode($response, true);
if (!empty($response['url'])) {
$download['背景PSD下载'] = $response['url'];
}
}
if (preg_match('/data-id="' . $site_code['1'] . '".*?class="b-down-operate /', $html) > 0) {
$http_curl = new HttpCurl('http://www.51yuansu.com/index.php?m=ajax&a=bdown&id=' . $site_code['1'], 'GET', null, $header, true, $this->cookie['cookie_id']);
$response = $http_curl->send_request()->get_response_body();
$response = json_decode($response, true);
if (!empty($response['url'])) {
$download['背景JPG下载'] = $response['url'];
}
}
if (!empty($download)) {
return ['code' => 1, 'site_code_type' => '', 'site_code' => $site_code['1'], 'msg' => $download];
}
return ['code' => 0, 'msg' => '解析失败，请联系管理员'];
}对 curl 进行 封装
<?php
namespace app\common\model;
use think\facade\Env;
class HttpCurl
{
public $request_url;
public $request_method;
public $request_header;
public $request_postfields;
public $request_gzip;
public $request_curlopts;
public $cookie_file;
public $timeout = 10;
private $response_data;
private $response_header;
private $response_body;
public function __construct($url = '', $method, $postfields = null, $headers = [], $gzip = false, $cookie_id = 0)
{
$this->request_url = $url;
$this->request_method = $method;
$this->request_header = array_merge([
'Accept-Encoding' => 'gzip, deflate',
'Accept-Language' => 'zh-CN,zh;q=0.9',
'Connection' => 'keep-alive',
'Upgrade-Insecure-Requests' => '1',
], $headers);
$this->request_postfields = $postfields;
$this->request_gzip = $gzip;
$cookie_file = Env::get('runtime_path') . 'site_cookie/cookie_' . ($cookie_id ?? 0);
if (is_file($cookie_file)) {
$this->cookie_file = $cookie_file;
}
}
public function request_url($url = '')
{
$this->request_url = $url;
return $this;
}
public function request_method($method = '')
{
$this->request_method = $method;
return $this;
}
public function request_header($headers = '')
{
$this->request_header = $headers;
return $this;
}
public function request_postfields($postfields = null)
{
$this->request_postfields = $postfields;
return $this;
}
public function request_gzip($gzip = '')
{
$this->request_gzip = $gzip;
return $this;
}
public function request_curlopts($curlopts = '')
{
$this->request_curlopts = $curlopts;
return $this;
}
public function request_cookie_file($cookie_file = '')
{
$this->cookie_file = $cookie_file;
return $this;
}
public function timeout($time = 10)
{
$this->timeout = $time;
return $this;
}
private function prep_request()
{
$header = [];
foreach ($this->request_header as $key => $value) {
if (!empty($this->cookie_file) && is_file($this->cookie_file) && $key == 'Cookie') {
continue;
}
$header[] = $key . ': ' . $value;
}
$curl = curl_init();
curl_setopt($curl, CURLOPT_TIMEOUT, $this->timeout);
curl_setopt($curl, CURLOPT_URL, $this->request_url);
curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($curl, CURLOPT_HEADER, true);
if (!empty($this->cookie_file) && is_file($this->cookie_file)) {
curl_setopt($curl, CURLOPT_COOKIEJAR, $this->cookie_file);
curl_setopt($curl, CURLOPT_COOKIEFILE, $this->cookie_file);
}
curl_setopt($curl, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36");
switch ($this->request_method) {
case 'POST':
curl_setopt($curl, CURLOPT_POST, true);
if (!empty($this->request_postfields)) {
curl_setopt($curl, CURLOPT_POSTFIELDS, is_array($this->request_postfields) ? http_build_query($this->request_postfields) : $this->request_postfields);
}
break;
default:
curl_setopt($curl, CURLOPT_CUSTOMREQUEST, $this->request_method);
break;
}
curl_setopt($curl, CURLOPT_HTTPHEADER, $header);
if ($this->request_gzip == true) {
curl_setopt($curl, CURLOPT_ENCODING, 'gzip');
}
curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
if (isset($this->request_curlopts) && sizeof($this->request_curlopts) > 0) {
foreach ($this->request_curlopts as $k => $v) {
curl_setopt($curl, $k, $v);
}
}
return $curl;
}
public function send_request()
{
$curl = $this->prep_request();
try {
$this->response_data = curl_exec($curl);
if (is_resource($curl)) {
$header_size = curl_getinfo($curl, CURLINFO_HEADER_SIZE);
$this->response_header = substr($this->response_data, 0, $header_size);
$this->response_body = substr($this->response_data, $header_size);
$this->response_code = curl_getinfo($curl, CURLINFO_HTTP_CODE);
$this->response_effective_url = curl_getinfo($curl, CURLINFO_EFFECTIVE_URL);
$this->response_info = curl_getinfo($curl);
$this->response_header = explode("\r\n\r\n", trim($this->response_header));
$this->response_header = array_pop($this->response_header);
$this->response_header = explode("\r\n", $this->response_header);
array_shift($this->response_header);
$header_assoc = [];
foreach ($this->response_header as $header) {
$kv = explode(': ', $header);
$header_assoc[strtolower($kv[0])] = isset($kv[1]) ? $kv[1] : '';
}
$this->response_header = $header_assoc;
$this->response_header['info'] = $this->response_info;
$this->response_header['info']['method'] = $this->request_method;
$this->response_header['info']['effective_url'] = $this->response_effective_url;
}
curl_close($curl);
} catch (\Exception $e) {
}
return $this;
}
public function get_response_data()
{
return $this->response_data;
}
public function get_response_header($header = null)
{
if ($header) {
return $this->response_header[strtolower($header)];
}
return $this->response_header;
}
public function get_response_body()
{
return $this->response_body;
}
}总结
- 项目基本实现了对各大素材网站统一进行处理，利用cookie 模拟登录获取素材。但是需要定期维护cookie。这个通过技术手段可以优化。
- 在以后的工作中，应多考虑需求的实现，会不会被curl 利用并攻击。
- 本次分享只是站在技术层面的分享，也仅限技术内部分享。
共 0 条评论关于 “PHP结合Curl实现资源共享”