初始版本发布
This commit is contained in:
杨志
216
app/service/CrawlerService.php
Normal file
216
app/service/CrawlerService.php
Normal file
@@ -0,0 +1,216 @@
|
||||
<?php
|
||||
declare (strict_types = 1);
|
||||
|
||||
namespace app\service;
|
||||
|
||||
/**
|
||||
* 爬虫服务类
|
||||
* 用于处理职位信息爬取相关逻辑
|
||||
*/
|
||||
class CrawlerService
|
||||
{
|
||||
/**
|
||||
* 基础URL
|
||||
*/
|
||||
private $baseUrl = 'http://gzrsks.oumakspt.com:62/tyzpwb';
|
||||
|
||||
/**
|
||||
* 获取职位树数据
|
||||
* @param string $dsdm 地区代码
|
||||
* @param array $cookies Cookie数据
|
||||
* @return array
|
||||
*/
|
||||
public function getPositionTree(string $dsdm, array $cookies): array
|
||||
{
|
||||
$url = $this->baseUrl . '/tree/getPositionTree.htm';
|
||||
|
||||
$cookieString = $this->buildCookieString($cookies);
|
||||
|
||||
$ch = curl_init();
|
||||
curl_setopt($ch, CURLOPT_URL, $url);
|
||||
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
||||
curl_setopt($ch, CURLOPT_HTTPHEADER, [
|
||||
'Cookie: ' . $cookieString,
|
||||
'Content-Type: application/x-www-form-urlencoded',
|
||||
]);
|
||||
curl_setopt($ch, CURLOPT_POST, true);
|
||||
curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query(['dsdm' => $dsdm]));
|
||||
curl_setopt($ch, CURLOPT_TIMEOUT, 30);
|
||||
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
|
||||
|
||||
$response = curl_exec($ch);
|
||||
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||
$error = curl_error($ch);
|
||||
curl_close($ch);
|
||||
|
||||
if ($error) {
|
||||
throw new \Exception('请求失败: ' . $error);
|
||||
}
|
||||
|
||||
if ($httpCode !== 200) {
|
||||
throw new \Exception('请求失败，HTTP状态码: ' . $httpCode);
|
||||
}
|
||||
|
||||
$data = json_decode($response, true);
|
||||
|
||||
if (json_last_error() !== JSON_ERROR_NONE) {
|
||||
throw new \Exception('JSON解析失败: ' . json_last_error_msg());
|
||||
}
|
||||
|
||||
return $data ?: [];
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取职位详细信息
|
||||
* @param string $zwdm 职位代码
|
||||
* @param string $examid 考试ID
|
||||
* @param array $cookies Cookie数据
|
||||
* @return array
|
||||
*/
|
||||
public function getPositionInfo(string $zwdm, string $examid, array $cookies): array
|
||||
{
|
||||
$url = $this->baseUrl . '/stuchooseexam/getPositionInfo.htm';
|
||||
|
||||
$cookieString = $this->buildCookieString($cookies);
|
||||
|
||||
$ch = curl_init();
|
||||
curl_setopt($ch, CURLOPT_URL, $url);
|
||||
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
||||
curl_setopt($ch, CURLOPT_HTTPHEADER, [
|
||||
'Cookie: ' . $cookieString,
|
||||
'Content-Type: application/x-www-form-urlencoded',
|
||||
]);
|
||||
curl_setopt($ch, CURLOPT_POST, true);
|
||||
curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query([
|
||||
'zwdm' => $zwdm,
|
||||
'examid' => $examid
|
||||
]));
|
||||
curl_setopt($ch, CURLOPT_TIMEOUT, 30);
|
||||
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
|
||||
|
||||
$response = curl_exec($ch);
|
||||
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||
$error = curl_error($ch);
|
||||
curl_close($ch);
|
||||
|
||||
if ($error) {
|
||||
throw new \Exception('请求失败: ' . $error);
|
||||
}
|
||||
|
||||
if ($httpCode !== 200) {
|
||||
throw new \Exception('请求失败，HTTP状态码: ' . $httpCode);
|
||||
}
|
||||
|
||||
$data = json_decode($response, true);
|
||||
|
||||
if (json_last_error() !== JSON_ERROR_NONE) {
|
||||
throw new \Exception('JSON解析失败: ' . json_last_error_msg());
|
||||
}
|
||||
|
||||
return $data ?: [];
|
||||
}
|
||||
|
||||
/**
|
||||
* 批量获取职位信息
|
||||
* @param array $zwdmList 职位代码列表
|
||||
* @param string $examid 考试ID
|
||||
* @param array $cookies Cookie数据
|
||||
* @return array
|
||||
*/
|
||||
public function batchGetPositionInfo(array $zwdmList, string $examid, array $cookies): array
|
||||
{
|
||||
$results = [];
|
||||
|
||||
foreach ($zwdmList as $zwdm) {
|
||||
try {
|
||||
$info = $this->getPositionInfo($zwdm, $examid, $cookies);
|
||||
|
||||
if (!empty($info)) {
|
||||
// 处理单条数据或数组数据
|
||||
if (isset($info[0])) {
|
||||
$item = $info[0];
|
||||
} else {
|
||||
$item = $info;
|
||||
}
|
||||
|
||||
// 计算竞争比
|
||||
$zprs = isset($item['zprs']) ? intval($item['zprs']) : 0;
|
||||
$bkrs = isset($item['bkrs']) ? intval($item['bkrs']) : 0;
|
||||
$competitionRatio = $zprs > 0 ? ($bkrs / $zprs) : 0;
|
||||
|
||||
$results[] = [
|
||||
'sbmc' => $item['sbmc'] ?? '', // 省份
|
||||
'dsmc' => $item['dsmc'] ?? '', // 地区
|
||||
'zpdwmc' => $item['zpdwmc'] ?? '', // 招聘单位/用人司局
|
||||
'zwmc' => $item['zwmc'] ?? '', // 职位名称
|
||||
'zwdm' => $item['zwdm'] ?? $zwdm, // 职位代码
|
||||
'zprs' => $zprs, // 招聘人数
|
||||
'bkrs' => $bkrs, // 审核通过人数
|
||||
'competition_ratio' => number_format($competitionRatio, 2), // 竞争比
|
||||
];
|
||||
}
|
||||
|
||||
// 避免请求过快，添加小延迟
|
||||
usleep(200000); // 0.2秒
|
||||
|
||||
} catch (\Exception $e) {
|
||||
// 记录错误但继续处理其他数据
|
||||
$results[] = [
|
||||
'zwdm' => $zwdm,
|
||||
'error' => $e->getMessage(),
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
return $results;
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建Cookie字符串
|
||||
* @param array $cookies Cookie数组
|
||||
* @return string
|
||||
*/
|
||||
private function buildCookieString(array $cookies): string
|
||||
{
|
||||
$cookieArray = [];
|
||||
|
||||
// 处理嵌套的Cookie结构
|
||||
if (isset($cookies['请求 Cookie'])) {
|
||||
$cookieData = $cookies['请求 Cookie'];
|
||||
} else {
|
||||
$cookieData = $cookies;
|
||||
}
|
||||
|
||||
foreach ($cookieData as $key => $value) {
|
||||
$cookieArray[] = $key . '=' . $value;
|
||||
}
|
||||
|
||||
return implode('; ', $cookieArray);
|
||||
}
|
||||
|
||||
/**
|
||||
* 从HTML中提取地区选项
|
||||
* @param string $html HTML内容
|
||||
* @return array
|
||||
*/
|
||||
public function extractDsdmOptions(string $html): array
|
||||
{
|
||||
$options = [];
|
||||
|
||||
// 使用正则表达式提取option标签
|
||||
preg_match_all('/<option\s+value="([^"]+)"[^>]*>([^<]+)<\/option>/i', $html, $matches);
|
||||
|
||||
if (!empty($matches[1])) {
|
||||
foreach ($matches[1] as $index => $value) {
|
||||
if (!empty($value)) { // 跳过"请选择"等空值
|
||||
$options[] = [
|
||||
'value' => $value,
|
||||
'text' => $matches[2][$index] ?? $value,
|
||||
];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $options;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user