diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..22a844d --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,31 @@ +# Repository Guidelines + +## Project Structure & Module Organization + +This is a C++11/CMake face-analysis application. `app/` owns UI orchestration, `network_camera_receiver/` captures HTTP MJPEG frames, and `face_pipeline/` contains motion gating, tracking, alignment, embedding, and publishing. `libfacedetection/` is the shared CNN detector. Store fixtures in `resources/`, tests in `tests/`, and pinned inference assets in `models/`. Treat `build/` as generated output. + +## Build, Test, and Development Commands + +Install CMake 3.10+, a C++11 compiler, and OpenCV development headers. Then use: + +```bash +cmake -S . -B build -DENABLE_AVX2=ON +cmake --build build -j +./build/app/face_detection_app +``` + +The application expects an MJPEG stream at `http://127.0.0.1:5000/video`; press Esc to exit. SIMD options are mutually exclusive. For AVX512 or ARM builds, disable AVX2 and enable the target option, for example `-DENABLE_AVX2=OFF -DENABLE_NEON=ON`. OpenMP is detected automatically. + +Run `ctest --test-dir build --output-on-failure` after building. For live pipeline changes, also confirm frame reception, stable track IDs, first-event-only feature logs, and clean shutdown against an MJPEG stream. + +## Coding Style & Naming Conventions + +Follow the existing C++ style: four-space indentation, opening braces on the same line, and focused comments for non-obvious buffer layouts or concurrency. Use `PascalCase` for classes (`NetworkCameraReceiver`), `camelCase` for functions (`getLatestFrame`), and `snake_case` for local variables (`window_name`). Keep public declarations in `include/` and implementations in `src/`. Preserve const-correctness and use RAII/standard synchronization primitives for new resource or thread ownership. There is no configured formatter or linter, so match adjacent code closely. + +## Testing Guidelines + +Integrate coverage with CTest and name files after behavior, such as `network_camera_receiver_test.cpp`. Avoid tests that require a physical camera; prefer fixtures, synthetic frames, fake clocks, or a controllable local stream. + +## Commit & Pull Request Guidelines + +Recent history uses short, lowercase, imperative summaries such as `add readme`. Keep commits focused and describe the user-visible change in the subject. Pull requests should explain scope, build/test commands run, platform and SIMD option used, and any stream assumptions. Link related issues and include screenshots for changes to rendered detection output. diff --git a/CMakeLists.txt b/CMakeLists.txt index 5581ab1..fc6242d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,6 +13,17 @@ endif() # 默认导出 compile_commands.json（便于 IDE/clangd 索引） set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +set(SFACE_MODEL "${CMAKE_SOURCE_DIR}/models/face_recognition_sface_2021dec.onnx") +set(SFACE_MODEL_SHA256 "0ba9fbfa01b5270c96627c4ef784da859931e02f04419c829e83484087c34e79") +if(EXISTS "${SFACE_MODEL}") + file(SHA256 "${SFACE_MODEL}" SFACE_MODEL_ACTUAL_SHA256) + if(NOT SFACE_MODEL_ACTUAL_SHA256 STREQUAL SFACE_MODEL_SHA256) + message(FATAL_ERROR "Bundled SFace model checksum mismatch") + endif() +else() + message(FATAL_ERROR "Missing bundled SFace model: ${SFACE_MODEL}") +endif() + # ============================================================================ # 0. SIMD 加速选项（作用于 libfacedetection 动态库） # X86/X64 CPU: cmake -DENABLE_AVX2=ON 或 -DENABLE_AVX512=ON @@ -64,4 +75,10 @@ message(STATUS "Threads library enabled.") # ============================================================================ add_subdirectory(libfacedetection) add_subdirectory(network_camera_receiver) +add_subdirectory(face_pipeline) add_subdirectory(app) + +include(CTest) +if(BUILD_TESTING) + add_subdirectory(tests) +endif() diff --git a/README.md b/README.md index 5f6d4b8..b1096b9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # FaceRecognition — 实时网络视频流人脸检测 -基于 CNN 的人脸检测应用：从 HTTP 网络视频流实时抓帧，调用 `libfacedetection` 进行人脸检测与关键点定位，并在窗口中绘制结果。 +基于 CNN 的人脸检测与特征提取应用：从 HTTP 网络视频流实时抓帧，经运动粗筛后检测人脸与关键点，完成对齐、SFace 特征提取和 IoU 跟踪。 - 作者：Liu zhenyu - 语言标准：C++11 @@ -27,23 +27,30 @@ FaceRecognition/ │ ├── CMakeLists.txt │ ├── include/network_camera_receiver.h │ └── src/network_camera_receiver.cpp +├── face_pipeline/ # 运动检测、跟踪、对齐、特征与消息接口 +├── models/ # 固定版本 SFace ONNX、许可证与校验信息 +├── tests/ # CTest 离线单元/模型测试 └── resources/ # 测试资源 ``` -### 三个子模块 +### 主要模块 | 模块 | 类型 | 职责 | |------|------|------| | `libfacedetection` | SHARED 动态库 | CNN 人脸检测（含 5 个关键点），仅导出 `facedetect_cnn` | | `network_camera_receiver` | STATIC 静态库 | 后台线程从 HTTP 视频流持续抓取最新帧，不包含检测逻辑 | -| `app` | 可执行程序 `face_detection_app` | 编排：拉帧 → 检测 → 绘制 → 显示 | +| `face_pipeline` | STATIC 静态库 | 运动粗筛、检测、IoU 跟踪、对齐、SFace 特征和结果接口 | +| `app` | 可执行程序 `face_detection_app` | 拉帧、提交任务并显示最新跟踪结果 | --- ## 功能特性 -- **实时检测**：后台线程抓帧，主线程检测显示，互不阻塞。 +- **低延迟流水线**：采集与检测分线程，容量为 1 的 latest-wins mailbox 不积压旧帧。 +- **运动粗筛**：有运动时触发检测；有人脸时每秒保活，无 track 时每 5 秒兜底扫描。 - **CNN 人脸检测**：输出人脸框 + 置信度 + 5 个面部关键点（双眼、鼻尖、嘴角）。 +- **特征提取**：五点相似变换对齐后，由 OpenCV SFace 输出 L2 归一化的 128 维向量。 +- **简单跟踪**：IoU 分配 `track_id`，每个 track 仅首次通过 `ResultSink` 发布。 - **SIMD 加速**：支持 AVX2 / AVX512 / NEON，CMake 选项一键开关并自动配置编译标志。 - **OpenMP 加速**：卷积运算可选多线程并行。 - **置信度过滤**：仅显示置信度 > 60 的人脸，降低误检。 @@ -75,9 +82,10 @@ sudo apt-get install build-essential cmake libopencv-dev cd FaceRecognition cmake -S . -B build -DENABLE_AVX2=ON cmake --build build -j +ctest --test-dir build --output-on-failure ``` -生成的可执行文件：`build/app/face_detection_app` +生成的可执行文件：`build/app/face_detection_app`。配置阶段会校验 `models/face_recognition_sface_2021dec.onnx` 的 SHA-256。 ### 2. SIMD 加速选项 @@ -157,7 +165,7 @@ std::string host_ip = "127.0.0.1"; // 改为推流主机 IP ./build/app/face_detection_app ``` -窗口将实时显示检测到的人脸框、置信度及 5 个关键点。按 **ESC** 退出。 +也可把兼容的 SFace 模型路径作为第一个参数传入。窗口显示人脸框、关键点和 `track_id`，按 **ESC** 退出。新 track 首次提取成功时输出一行 `face_feature` 日志，只记录元数据和特征维度，不打印完整向量。 --- @@ -190,17 +198,11 @@ short* p = ((short*)(pResults + 1)) + FACEDETECTION_RESULT_STRIDE_SHORTS * i; ## 项目架构流程 ``` -┌─────────────────────┐ 最新帧 ┌──────────────────────┐ -│ network_camera_ │ ─────────> │ app (main.cpp) │ -│ receiver (STATIC) │ getLatest │ ┌──────────────────┐ │ -│ 后台线程抓帧 │ Frame() │ │ facedetect_cnn() │ │ -└─────────────────────┘ │ │ (libfacedetect)│ │ - ▲ │ └────────┬─────────┘ │ - │ HTTP MJPEG │ v │ - │ /video │ 绘制框/关键点/置信度 │ -┌───────┴──────────────┐ │ cv::imshow │ -│ 推流端 (Flask/摄像头) │ └───────────────────────┘ -└──────────────────────┘ +HTTP MJPEG → 抓帧线程 → 主线程运动粗筛 → latest-wins mailbox + ↓ + 检测 worker：人脸检测 → IoU 跟踪 + ↓（仅未发布 track） + 五点对齐 → SFace → ResultSink ``` --- diff --git a/app/CMakeLists.txt b/app/CMakeLists.txt index f190f63..48657bd 100644 --- a/app/CMakeLists.txt +++ b/app/CMakeLists.txt @@ -13,6 +13,7 @@ add_executable(face_detection_app target_link_libraries(face_detection_app PRIVATE facedetection network_camera_receiver + face_pipeline ${OpenCV_LIBS} ) diff --git a/app/src/main.cpp b/app/src/main.cpp index 33fc4b8..ecb7371 100644 --- a/app/src/main.cpp +++ b/app/src/main.cpp @@ -1,98 +1,80 @@ -/* - * @Author: Liu zhenyu - * @Description: 主程序（重构版） - * 流程：1. 从 network_camera_receiver 取最新帧； - * 2. 调用 libfacedetection 进行人脸检测； - * 3. 绘制并显示检测结果。 - * libfacedetection 以独立动态库形式链接。 - */ +#include +#include +#include +#include #include -#include -#include +#include "face_pipeline/face_embedder.h" +#include "face_pipeline/face_pipeline.h" +#include "face_pipeline/result_sink.h" #include "network_camera_receiver.h" -#include "facedetectcnn.h" -// 将 libfacedetection 的检测结果绘制到帧上 -// pResults[0] = 人脸数量；其后每张脸占 FACEDETECTION_RESULT_STRIDE_SHORTS 个 short -static void drawFaces(cv::Mat& frame, const int* pResults) { - if (!pResults) return; - int faces = pResults[0]; +namespace { - for (int i = 0; i < faces; i++) { - // 每个人脸结果占用 FACEDETECTION_RESULT_STRIDE_SHORTS(=16) 个 short： - // p[0]=置信度(score*100), p[1..4]=x/y/w/h, - // p[5..14]=5个关键点(x,y交替), p[15]=对齐填充 - short* p = ((short*)(pResults + 1)) + FACEDETECTION_RESULT_STRIDE_SHORTS * i; - - int confidence = p[0]; - int x = p[1]; - int y = p[2]; - int w = p[3]; - int h = p[4]; - - // 只显示置信度大于 60 的结果以过滤误检 - if (confidence > 60) { - cv::rectangle(frame, cv::Rect(x, y, w, h), cv::Scalar(0, 255, 0), 2); - - // 置信度标注 - cv::putText(frame, std::to_string(confidence), - cv::Point(x, y - 5), cv::FONT_HERSHEY_SIMPLEX, - 0.5, cv::Scalar(0, 255, 0), 1); - - // 五个特征点 - cv::circle(frame, cv::Point(p[5], p[6]), 2, cv::Scalar(255, 0, 0), -1); - cv::circle(frame, cv::Point(p[7], p[8]), 2, cv::Scalar(0, 0, 255), -1); - cv::circle(frame, cv::Point(p[9], p[10]), 2, cv::Scalar(0, 255, 0), -1); - cv::circle(frame, cv::Point(p[11], p[12]), 2, cv::Scalar(255, 0, 255), -1); - cv::circle(frame, cv::Point(p[13], p[14]), 2, cv::Scalar(0, 255, 255), -1); +void drawTracks(cv::Mat& frame, const std::vector& tracks) { + for (const face_pipeline::TrackedFace& track : tracks) { + const face_pipeline::FaceObservation& face = track.observation; + cv::rectangle(frame, face.bbox, cv::Scalar(0, 255, 0), 2); + cv::putText(frame, "track " + std::to_string(track.track_id), + cv::Point(face.bbox.x, std::max(15, face.bbox.y - 5)), + cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 255, 0), 1); + for (const cv::Point2f& landmark : face.landmarks) { + cv::circle(frame, landmark, 2, cv::Scalar(0, 0, 255), -1); } } } -int main() { - // 替换为你查到的视频流宿主机 IP - std::string host_ip = "127.0.0.1"; +} // namespace - // 1) 接收端：仅负责抓帧 - NetworkCameraReceiver receiver(host_ip, 5000); - if (!receiver.connect()) { - std::cerr << "连接视频流失败，退出。" << std::endl; - return -1; - } - receiver.start(); // 启动后台抓帧线程 +int main(int argc, char** argv) { + const std::string model_path = argc > 1 + ? argv[1] + : std::string(SOURCE_ROOT) + "/models/face_recognition_sface_2021dec.onnx"; - // libfacedetection 结果缓冲区（大小由库定义的宏决定） - unsigned char* pBuffer = new unsigned char[FACEDETECTION_RESULT_BUFFER_SIZE]; + try { + std::unique_ptr embedder( + new face_pipeline::SFaceEmbedder(model_path)); + std::unique_ptr sink( + new face_pipeline::LoggingResultSink(std::cout)); + face_pipeline::FacePipeline pipeline(std::move(embedder), std::move(sink)); - const std::string window_name = "Face Detection"; - cv::namedWindow(window_name, cv::WINDOW_AUTOSIZE); + NetworkCameraReceiver receiver("127.0.0.1", 5000); + if (!receiver.connect()) return 1; - std::cout << "人脸检测已启动，按 ESC 退出..." << std::endl; + pipeline.start(); + receiver.start(); - cv::Mat frame; - while (true) { - // ---- 步骤 1：接收最新帧 ---- - if (receiver.getLatestFrame(frame) && !frame.empty()) { - // ---- 步骤 2：调用 libfacedetection 检测 ---- - // 注意：OpenCV 采集到的帧为 BGR，正好符合 facedetect_cnn 的输入要求 - int* pResults = facedetect_cnn(pBuffer, - frame.data, - frame.cols, - frame.rows, - (int)frame.step); + const std::string window_name = "Face Detection and Features"; + cv::namedWindow(window_name, cv::WINDOW_AUTOSIZE); + std::cout << "Face pipeline started; press ESC to exit." << std::endl; - // ---- 步骤 3：显示检测结果 ---- - drawFaces(frame, pResults); - cv::imshow(window_name, frame); + NetworkFrame network_frame; + while (true) { + if (receiver.getLatestFrame(network_frame) && !network_frame.image.empty()) { + face_pipeline::FramePacket packet; + packet.frame = network_frame.image; + packet.frame_id = network_frame.frame_id; + packet.captured_at = network_frame.captured_at; + packet.captured_at_unix_ms = network_frame.captured_at_unix_ms; + pipeline.submit(packet); + + cv::Mat display = network_frame.image.clone(); + drawTracks(display, pipeline.latestTracks()); + cv::imshow(window_name, display); + } + if (cv::waitKey(30) == 27) break; } - if (cv::waitKey(30) == 27) break; // ESC 退出 + receiver.stop(); + pipeline.stop(); + cv::destroyAllWindows(); + } catch (const cv::Exception& error) { + std::cerr << "OpenCV initialization failed: " << error.what() << std::endl; + return 1; + } catch (const std::exception& error) { + std::cerr << "Initialization failed: " << error.what() << std::endl; + return 1; } - - receiver.stop(); - delete[] pBuffer; - cv::destroyAllWindows(); return 0; } diff --git a/face_pipeline/CMakeLists.txt b/face_pipeline/CMakeLists.txt new file mode 100644 index 0000000..79c82a8 --- /dev/null +++ b/face_pipeline/CMakeLists.txt @@ -0,0 +1,25 @@ +add_library(face_pipeline STATIC + src/motion_detector.cpp + src/face_detector.cpp + src/face_aligner.cpp + src/face_embedder.cpp + src/iou_tracker.cpp + src/result_sink.cpp + src/face_pipeline.cpp +) + +target_include_directories(face_pipeline PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/include +) + +target_link_libraries(face_pipeline PUBLIC + facedetection + ${OpenCV_LIBS} + Threads::Threads +) + +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(face_pipeline PRIVATE -O3) +elseif(MSVC) + target_compile_options(face_pipeline PRIVATE /O2) +endif() diff --git a/face_pipeline/include/face_pipeline/face_aligner.h b/face_pipeline/include/face_pipeline/face_aligner.h new file mode 100644 index 0000000..831e8b5 --- /dev/null +++ b/face_pipeline/include/face_pipeline/face_aligner.h @@ -0,0 +1,15 @@ +#pragma once + +#include + +#include "face_pipeline/types.h" + +namespace face_pipeline { + +class FaceAligner { +public: + static cv::Size outputSize(); + bool align(const cv::Mat& frame, const FaceObservation& face, cv::Mat& aligned) const; +}; + +} // namespace face_pipeline diff --git a/face_pipeline/include/face_pipeline/face_detector.h b/face_pipeline/include/face_pipeline/face_detector.h new file mode 100644 index 0000000..7022a5b --- /dev/null +++ b/face_pipeline/include/face_pipeline/face_detector.h @@ -0,0 +1,21 @@ +#pragma once + +#include + +#include + +#include "face_pipeline/types.h" + +namespace face_pipeline { + +class FaceDetector { +public: + explicit FaceDetector(float minimum_score = 0.60f); + std::vector detect(const cv::Mat& frame); + +private: + float minimum_score_; + std::vector result_buffer_; +}; + +} // namespace face_pipeline diff --git a/face_pipeline/include/face_pipeline/face_embedder.h b/face_pipeline/include/face_pipeline/face_embedder.h new file mode 100644 index 0000000..3c4fb44 --- /dev/null +++ b/face_pipeline/include/face_pipeline/face_embedder.h @@ -0,0 +1,30 @@ +#pragma once + +#include +#include +#include + +#include + +namespace face_pipeline { + +class FaceEmbedder { +public: + virtual ~FaceEmbedder() = default; + virtual std::vector extract(const cv::Mat& aligned_face) = 0; + virtual std::string modelName() const = 0; +}; + +class SFaceEmbedder : public FaceEmbedder { +public: + explicit SFaceEmbedder(const std::string& model_path); + ~SFaceEmbedder() override; + std::vector extract(const cv::Mat& aligned_face) override; + std::string modelName() const override; + +private: + class Impl; + std::unique_ptr impl_; +}; + +} // namespace face_pipeline diff --git a/face_pipeline/include/face_pipeline/face_pipeline.h b/face_pipeline/include/face_pipeline/face_pipeline.h new file mode 100644 index 0000000..3bdad38 --- /dev/null +++ b/face_pipeline/include/face_pipeline/face_pipeline.h @@ -0,0 +1,62 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "face_pipeline/face_aligner.h" +#include "face_pipeline/face_detector.h" +#include "face_pipeline/face_embedder.h" +#include "face_pipeline/iou_tracker.h" +#include "face_pipeline/motion_detector.h" +#include "face_pipeline/result_sink.h" +#include "face_pipeline/types.h" + +namespace face_pipeline { + +class FacePipeline { +public: + FacePipeline(std::unique_ptr embedder, + std::unique_ptr sink); + ~FacePipeline(); + + void start(); + void stop(); + bool submit(const FramePacket& packet); + std::vector latestTracks() const; + +private: + void workerLoop(); + void process(const FramePacket& packet); + + MotionDetector motion_detector_; + FaceDetector face_detector_; + FaceAligner face_aligner_; + IouTracker tracker_; + std::unique_ptr embedder_; + std::unique_ptr sink_; + + mutable std::mutex state_mutex_; + std::vector latest_tracks_; + std::set published_tracks_; + + std::mutex mailbox_mutex_; + std::condition_variable mailbox_cv_; + FramePacket mailbox_; + bool has_mail_ = false; + bool running_ = false; + std::atomic has_active_tracks_{false}; + std::atomic submitted_frames_{0}; + std::atomic overwritten_frames_{0}; + std::uint64_t processed_frames_ = 0; + std::uint64_t alignment_failures_ = 0; + std::uint64_t embedding_failures_ = 0; + std::uint64_t publish_failures_ = 0; + std::thread worker_; +}; + +} // namespace face_pipeline diff --git a/face_pipeline/include/face_pipeline/iou_tracker.h b/face_pipeline/include/face_pipeline/iou_tracker.h new file mode 100644 index 0000000..8fe0c00 --- /dev/null +++ b/face_pipeline/include/face_pipeline/iou_tracker.h @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include + +#include "face_pipeline/types.h" + +namespace face_pipeline { + +struct IouTrackerConfig { + float match_threshold = 0.30f; + std::chrono::milliseconds track_ttl{2000}; +}; + +class IouTracker { +public: + explicit IouTracker(const IouTrackerConfig& config = IouTrackerConfig()); + std::vector update(const std::vector& observations, + SteadyTime now); + bool hasActiveTracks(SteadyTime now) const; + std::vector activeTrackIds() const; + std::size_t size() const; + +private: + struct Track { + std::uint64_t id; + FaceObservation observation; + SteadyTime last_seen; + }; + + static float intersectionOverUnion(const cv::Rect& a, const cv::Rect& b); + void expire(SteadyTime now); + + IouTrackerConfig config_; + std::vector tracks_; + std::uint64_t next_track_id_ = 1; +}; + +} // namespace face_pipeline diff --git a/face_pipeline/include/face_pipeline/motion_detector.h b/face_pipeline/include/face_pipeline/motion_detector.h new file mode 100644 index 0000000..adeb6f6 --- /dev/null +++ b/face_pipeline/include/face_pipeline/motion_detector.h @@ -0,0 +1,34 @@ +#pragma once + +#include + +#include + +#include "face_pipeline/types.h" + +namespace face_pipeline { + +struct MotionDetectorConfig { + cv::Size analysis_size{320, 180}; + int pixel_threshold = 20; + double changed_ratio_threshold = 0.01; + std::chrono::milliseconds active_hold{500}; + std::chrono::milliseconds tracked_scan_interval{1000}; + std::chrono::milliseconds idle_scan_interval{5000}; +}; + +class MotionDetector { +public: + explicit MotionDetector(const MotionDetectorConfig& config = MotionDetectorConfig()); + bool shouldProcess(const cv::Mat& frame, SteadyTime now, bool has_active_tracks); + void reset(); + +private: + MotionDetectorConfig config_; + cv::Mat previous_gray_; + SteadyTime last_motion_; + SteadyTime last_request_; + bool initialized_ = false; +}; + +} // namespace face_pipeline diff --git a/face_pipeline/include/face_pipeline/result_sink.h b/face_pipeline/include/face_pipeline/result_sink.h new file mode 100644 index 0000000..eac3c62 --- /dev/null +++ b/face_pipeline/include/face_pipeline/result_sink.h @@ -0,0 +1,24 @@ +#pragma once + +#include + +#include "face_pipeline/types.h" + +namespace face_pipeline { + +class ResultSink { +public: + virtual ~ResultSink() = default; + virtual bool publish(const FaceFeatureEvent& event) = 0; +}; + +class LoggingResultSink : public ResultSink { +public: + explicit LoggingResultSink(std::ostream& output); + bool publish(const FaceFeatureEvent& event) override; + +private: + std::ostream& output_; +}; + +} // namespace face_pipeline diff --git a/face_pipeline/include/face_pipeline/types.h b/face_pipeline/include/face_pipeline/types.h new file mode 100644 index 0000000..a45da7a --- /dev/null +++ b/face_pipeline/include/face_pipeline/types.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace face_pipeline { + +using SteadyTime = std::chrono::steady_clock::time_point; + +struct FramePacket { + cv::Mat frame; + std::uint64_t frame_id = 0; + SteadyTime captured_at; + std::int64_t captured_at_unix_ms = 0; +}; + +struct FaceObservation { + cv::Rect bbox; + float score = 0.0f; + std::array landmarks; +}; + +struct TrackedFace { + std::uint64_t track_id = 0; + FaceObservation observation; + bool is_new = false; +}; + +struct FaceFeatureEvent { + std::uint64_t track_id = 0; + std::uint64_t frame_id = 0; + std::int64_t captured_at_unix_ms = 0; + FaceObservation observation; + std::string embedding_model; + std::vector embedding; +}; + +} // namespace face_pipeline diff --git a/face_pipeline/src/face_aligner.cpp b/face_pipeline/src/face_aligner.cpp new file mode 100644 index 0000000..4d33370 --- /dev/null +++ b/face_pipeline/src/face_aligner.cpp @@ -0,0 +1,41 @@ +#include "face_pipeline/face_aligner.h" + +#include +#include + +#include +#include + +namespace face_pipeline { +namespace { + +const std::vector kSFaceTemplate = { + {38.2946f, 51.6963f}, {73.5318f, 51.5014f}, {56.0252f, 71.7366f}, + {41.5493f, 92.3655f}, {70.7299f, 92.2041f} +}; + +} // namespace + +cv::Size FaceAligner::outputSize() { + return cv::Size(112, 112); +} + +bool FaceAligner::align(const cv::Mat& frame, const FaceObservation& face, + cv::Mat& aligned) const { + if (frame.empty() || face.bbox.width < 12 || face.bbox.height < 12) return false; + + std::vector source(face.landmarks.begin(), face.landmarks.end()); + for (const cv::Point2f& point : source) { + if (!std::isfinite(point.x) || !std::isfinite(point.y)) return false; + } + + cv::Mat inliers; + cv::Mat transform = cv::estimateAffinePartial2D(source, kSFaceTemplate, inliers, cv::LMEDS); + if (transform.empty() || transform.rows != 2 || transform.cols != 3) return false; + + cv::warpAffine(frame, aligned, transform, outputSize(), cv::INTER_LINEAR, + cv::BORDER_CONSTANT, cv::Scalar()); + return !aligned.empty(); +} + +} // namespace face_pipeline diff --git a/face_pipeline/src/face_detector.cpp b/face_pipeline/src/face_detector.cpp new file mode 100644 index 0000000..0be20e7 --- /dev/null +++ b/face_pipeline/src/face_detector.cpp @@ -0,0 +1,42 @@ +#include "face_pipeline/face_detector.h" + +#include + +#include "facedetectcnn.h" + +namespace face_pipeline { + +FaceDetector::FaceDetector(float minimum_score) + : minimum_score_(minimum_score), result_buffer_(FACEDETECTION_RESULT_BUFFER_SIZE) {} + +std::vector FaceDetector::detect(const cv::Mat& frame) { + std::vector observations; + if (frame.empty() || frame.type() != CV_8UC3) return observations; + + int* results = facedetect_cnn(result_buffer_.data(), frame.data, frame.cols, frame.rows, + static_cast(frame.step)); + if (!results) return observations; + + observations.reserve(static_cast(std::max(0, results[0]))); + for (int i = 0; i < results[0]; ++i) { + short* values = reinterpret_cast(results + 1) + + FACEDETECTION_RESULT_STRIDE_SHORTS * i; + const float score = static_cast(values[0]) / 100.0f; + cv::Rect bbox(values[1], values[2], values[3], values[4]); + bbox &= cv::Rect(0, 0, frame.cols, frame.rows); + if (score <= minimum_score_ || bbox.width <= 0 || bbox.height <= 0) continue; + + FaceObservation observation; + observation.bbox = bbox; + observation.score = score; + for (std::size_t landmark = 0; landmark < observation.landmarks.size(); ++landmark) { + observation.landmarks[landmark] = cv::Point2f( + static_cast(values[5 + landmark * 2]), + static_cast(values[6 + landmark * 2])); + } + observations.push_back(observation); + } + return observations; +} + +} // namespace face_pipeline diff --git a/face_pipeline/src/face_embedder.cpp b/face_pipeline/src/face_embedder.cpp new file mode 100644 index 0000000..d80613c --- /dev/null +++ b/face_pipeline/src/face_embedder.cpp @@ -0,0 +1,59 @@ +#include "face_pipeline/face_embedder.h" + +#include +#include + +#include + +namespace face_pipeline { + +class SFaceEmbedder::Impl { +public: + explicit Impl(const std::string& path) + : recognizer(cv::FaceRecognizerSF::create(path, "")) { + if (recognizer.empty()) throw std::runtime_error("failed to load SFace model: " + path); + } + + cv::Ptr recognizer; +}; + +SFaceEmbedder::SFaceEmbedder(const std::string& model_path) + : impl_(new Impl(model_path)) { + cv::Mat validation_input(112, 112, CV_8UC3, cv::Scalar(127, 127, 127)); + extract(validation_input); +} + +SFaceEmbedder::~SFaceEmbedder() = default; + +std::vector SFaceEmbedder::extract(const cv::Mat& aligned_face) { + if (aligned_face.empty() || aligned_face.size() != cv::Size(112, 112) || + aligned_face.type() != CV_8UC3) { + throw std::runtime_error("SFace input must be a 112x112 BGR image"); + } + + cv::Mat feature; + impl_->recognizer->feature(aligned_face, feature); + cv::Mat flattened = feature.reshape(1, 1); + if (flattened.type() != CV_32F || flattened.total() != 128) { + throw std::runtime_error("SFace output must contain 128 float values"); + } + + const float norm = static_cast(cv::norm(flattened, cv::NORM_L2)); + if (!std::isfinite(norm) || norm <= 1e-12f) { + throw std::runtime_error("SFace produced an invalid feature vector"); + } + flattened /= norm; + + const float* begin = flattened.ptr(); + std::vector embedding(begin, begin + flattened.total()); + for (float value : embedding) { + if (!std::isfinite(value)) throw std::runtime_error("SFace produced non-finite values"); + } + return embedding; +} + +std::string SFaceEmbedder::modelName() const { + return "opencv_sface_2021dec"; +} + +} // namespace face_pipeline diff --git a/face_pipeline/src/face_pipeline.cpp b/face_pipeline/src/face_pipeline.cpp new file mode 100644 index 0000000..f509cf5 --- /dev/null +++ b/face_pipeline/src/face_pipeline.cpp @@ -0,0 +1,144 @@ +#include "face_pipeline/face_pipeline.h" + +#include +#include + +namespace face_pipeline { + +FacePipeline::FacePipeline(std::unique_ptr embedder, + std::unique_ptr sink) + : embedder_(std::move(embedder)), sink_(std::move(sink)) { + if (!embedder_ || !sink_) throw std::invalid_argument("pipeline dependencies are required"); +} + +FacePipeline::~FacePipeline() { + stop(); +} + +void FacePipeline::start() { + std::lock_guard lock(mailbox_mutex_); + if (running_) return; + running_ = true; + worker_ = std::thread(&FacePipeline::workerLoop, this); +} + +void FacePipeline::stop() { + { + std::lock_guard lock(mailbox_mutex_); + if (!running_) return; + running_ = false; + has_mail_ = false; + } + mailbox_cv_.notify_all(); + if (worker_.joinable()) worker_.join(); +} + +bool FacePipeline::submit(const FramePacket& packet) { + if (!motion_detector_.shouldProcess(packet.frame, packet.captured_at, + has_active_tracks_.load())) return false; + + std::lock_guard lock(mailbox_mutex_); + if (!running_) return false; + submitted_frames_.fetch_add(1); + if (has_mail_) overwritten_frames_.fetch_add(1); + mailbox_ = packet; + mailbox_.frame = packet.frame.clone(); + has_mail_ = true; + mailbox_cv_.notify_one(); + return true; +} + +std::vector FacePipeline::latestTracks() const { + std::lock_guard lock(state_mutex_); + return latest_tracks_; +} + +void FacePipeline::workerLoop() { + while (true) { + FramePacket packet; + { + std::unique_lock lock(mailbox_mutex_); + mailbox_cv_.wait(lock, [&] { return has_mail_ || !running_; }); + if (!running_) break; + packet = mailbox_; + has_mail_ = false; + } + process(packet); + } +} + +void FacePipeline::process(const FramePacket& packet) { + try { + ++processed_frames_; + const std::vector observations = face_detector_.detect(packet.frame); + const std::vector tracked = tracker_.update(observations, packet.captured_at); + has_active_tracks_.store(tracker_.hasActiveTracks(packet.captured_at)); + + { + std::lock_guard lock(state_mutex_); + latest_tracks_ = tracked; + const std::vector active_ids = tracker_.activeTrackIds(); + const std::set active(active_ids.begin(), active_ids.end()); + for (std::set::iterator it = published_tracks_.begin(); + it != published_tracks_.end();) { + if (active.count(*it) == 0) { + it = published_tracks_.erase(it); + } else { + ++it; + } + } + } + + for (const TrackedFace& face : tracked) { + { + std::lock_guard lock(state_mutex_); + if (published_tracks_.count(face.track_id) != 0) continue; + } + + try { + cv::Mat aligned; + if (!face_aligner_.align(packet.frame, face.observation, aligned)) { + ++alignment_failures_; + continue; + } + + FaceFeatureEvent event; + event.track_id = face.track_id; + event.frame_id = packet.frame_id; + event.captured_at_unix_ms = packet.captured_at_unix_ms; + event.observation = face.observation; + event.embedding_model = embedder_->modelName(); + event.embedding = embedder_->extract(aligned); + if (sink_->publish(event)) { + std::lock_guard lock(state_mutex_); + published_tracks_.insert(face.track_id); + } else { + ++publish_failures_; + } + } catch (const cv::Exception& error) { + ++embedding_failures_; + std::cerr << "feature extraction failed for track " << face.track_id + << ": " << error.what() << std::endl; + } catch (const std::exception& error) { + ++embedding_failures_; + std::cerr << "feature extraction failed for track " << face.track_id + << ": " << error.what() << std::endl; + } + } + + if (processed_frames_ % 100 == 0) { + std::clog << "face_pipeline_stats submitted=" << submitted_frames_.load() + << " overwritten=" << overwritten_frames_.load() + << " processed=" << processed_frames_ + << " align_failed=" << alignment_failures_ + << " embed_failed=" << embedding_failures_ + << " publish_failed=" << publish_failures_ << std::endl; + } + } catch (const cv::Exception& error) { + std::cerr << "face pipeline OpenCV error: " << error.what() << std::endl; + } catch (const std::exception& error) { + std::cerr << "face pipeline error: " << error.what() << std::endl; + } +} + +} // namespace face_pipeline diff --git a/face_pipeline/src/iou_tracker.cpp b/face_pipeline/src/iou_tracker.cpp new file mode 100644 index 0000000..61bbd70 --- /dev/null +++ b/face_pipeline/src/iou_tracker.cpp @@ -0,0 +1,92 @@ +#include "face_pipeline/iou_tracker.h" + +#include + +namespace face_pipeline { + +IouTracker::IouTracker(const IouTrackerConfig& config) : config_(config) {} + +float IouTracker::intersectionOverUnion(const cv::Rect& a, const cv::Rect& b) { + const cv::Rect intersection = a & b; + if (intersection.area() <= 0) return 0.0f; + const float union_area = static_cast(a.area() + b.area() - intersection.area()); + return union_area > 0.0f ? static_cast(intersection.area()) / union_area : 0.0f; +} + +void IouTracker::expire(SteadyTime now) { + tracks_.erase(std::remove_if(tracks_.begin(), tracks_.end(), [&](const Track& track) { + return now - track.last_seen >= config_.track_ttl; + }), tracks_.end()); +} + +std::vector IouTracker::update( + const std::vector& observations, SteadyTime now) { + expire(now); + + struct Candidate { + std::size_t track; + std::size_t observation; + float iou; + }; + std::vector candidates; + for (std::size_t track = 0; track < tracks_.size(); ++track) { + for (std::size_t observation = 0; observation < observations.size(); ++observation) { + const float iou = intersectionOverUnion(tracks_[track].observation.bbox, + observations[observation].bbox); + if (iou >= config_.match_threshold) candidates.push_back({track, observation, iou}); + } + } + std::sort(candidates.begin(), candidates.end(), [](const Candidate& left, + const Candidate& right) { + return left.iou > right.iou; + }); + + std::vector track_used(tracks_.size(), false); + std::vector observation_used(observations.size(), false); + std::vector result; + for (const Candidate& candidate : candidates) { + if (track_used[candidate.track] || observation_used[candidate.observation]) continue; + Track& track = tracks_[candidate.track]; + track.observation = observations[candidate.observation]; + track.last_seen = now; + track_used[candidate.track] = true; + observation_used[candidate.observation] = true; + TrackedFace tracked_face; + tracked_face.track_id = track.id; + tracked_face.observation = track.observation; + tracked_face.is_new = false; + result.push_back(tracked_face); + } + + for (std::size_t observation = 0; observation < observations.size(); ++observation) { + if (observation_used[observation]) continue; + Track track{next_track_id_++, observations[observation], now}; + tracks_.push_back(track); + TrackedFace tracked_face; + tracked_face.track_id = track.id; + tracked_face.observation = track.observation; + tracked_face.is_new = true; + result.push_back(tracked_face); + } + return result; +} + +bool IouTracker::hasActiveTracks(SteadyTime now) const { + for (const Track& track : tracks_) { + if (now - track.last_seen < config_.track_ttl) return true; + } + return false; +} + +std::vector IouTracker::activeTrackIds() const { + std::vector ids; + ids.reserve(tracks_.size()); + for (const Track& track : tracks_) ids.push_back(track.id); + return ids; +} + +std::size_t IouTracker::size() const { + return tracks_.size(); +} + +} // namespace face_pipeline diff --git a/face_pipeline/src/motion_detector.cpp b/face_pipeline/src/motion_detector.cpp new file mode 100644 index 0000000..76a9db4 --- /dev/null +++ b/face_pipeline/src/motion_detector.cpp @@ -0,0 +1,54 @@ +#include "face_pipeline/motion_detector.h" + +#include + +namespace face_pipeline { + +MotionDetector::MotionDetector(const MotionDetectorConfig& config) : config_(config) {} + +bool MotionDetector::shouldProcess(const cv::Mat& frame, SteadyTime now, + bool has_active_tracks) { + if (frame.empty()) return false; + + cv::Mat resized; + cv::Mat gray; + cv::resize(frame, resized, config_.analysis_size); + cv::cvtColor(resized, gray, cv::COLOR_BGR2GRAY); + cv::GaussianBlur(gray, gray, cv::Size(5, 5), 0.0); + + if (!initialized_) { + gray.copyTo(previous_gray_); + last_motion_ = now - config_.active_hold; + last_request_ = now; + initialized_ = true; + return false; + } + + cv::Mat difference; + cv::Mat changed; + cv::absdiff(previous_gray_, gray, difference); + cv::threshold(difference, changed, config_.pixel_threshold, 255, cv::THRESH_BINARY); + cv::morphologyEx(changed, changed, cv::MORPH_OPEN, + cv::getStructuringElement(cv::MORPH_RECT, cv::Size(3, 3))); + gray.copyTo(previous_gray_); + + const double changed_ratio = static_cast(cv::countNonZero(changed)) / + static_cast(changed.total()); + if (changed_ratio >= config_.changed_ratio_threshold) last_motion_ = now; + + const std::chrono::milliseconds scan_interval = has_active_tracks + ? config_.tracked_scan_interval : config_.idle_scan_interval; + const bool motion_active = now - last_motion_ < config_.active_hold; + const bool scan_due = now - last_request_ >= scan_interval; + if (!motion_active && !scan_due) return false; + + last_request_ = now; + return true; +} + +void MotionDetector::reset() { + previous_gray_.release(); + initialized_ = false; +} + +} // namespace face_pipeline diff --git a/face_pipeline/src/result_sink.cpp b/face_pipeline/src/result_sink.cpp new file mode 100644 index 0000000..134eadb --- /dev/null +++ b/face_pipeline/src/result_sink.cpp @@ -0,0 +1,19 @@ +#include "face_pipeline/result_sink.h" + +namespace face_pipeline { + +LoggingResultSink::LoggingResultSink(std::ostream& output) : output_(output) {} + +bool LoggingResultSink::publish(const FaceFeatureEvent& event) { + output_ << "face_feature track_id=" << event.track_id + << " frame_id=" << event.frame_id + << " captured_at_ms=" << event.captured_at_unix_ms + << " bbox=" << event.observation.bbox.x << ',' << event.observation.bbox.y + << ',' << event.observation.bbox.width << ',' << event.observation.bbox.height + << " score=" << event.observation.score + << " model=" << event.embedding_model + << " embedding_dim=" << event.embedding.size() << std::endl; + return static_cast(output_); +} + +} // namespace face_pipeline diff --git a/models/LICENSE b/models/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/models/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/models/README.md b/models/README.md new file mode 100644 index 0000000..5fc5ab3 --- /dev/null +++ b/models/README.md @@ -0,0 +1,18 @@ +# Face Embedding Model + +`face_recognition_sface_2021dec.onnx` is the OpenCV Zoo SFace model used by +the default application. It accepts an aligned 112 x 112 BGR face and returns +a 128-dimensional feature vector. The application L2-normalizes that vector +before publishing it. + +- Source: https://github.com/opencv/opencv_zoo/tree/main/models/face_recognition_sface +- Download mirror: https://huggingface.co/opencv/face_recognition_sface +- Upstream project: https://github.com/opencv/opencv +- Model filename: `face_recognition_sface_2021dec.onnx` +- SHA-256: `0ba9fbfa01b5270c96627c4ef784da859931e02f04419c829e83484087c34e79` +- License: Apache License 2.0; see `LICENSE` in this directory. + +Keep the license and model metadata with the model when redistributing the +repository. CMake verifies the bundled model checksum, and the application +validates its output contract during startup rather than running without feature +extraction. diff --git a/models/face_recognition_sface_2021dec.onnx b/models/face_recognition_sface_2021dec.onnx new file mode 100644 index 0000000..6767be0 Binary files /dev/null and b/models/face_recognition_sface_2021dec.onnx differ diff --git a/network_camera_receiver/include/network_camera_receiver.h b/network_camera_receiver/include/network_camera_receiver.h index 9e191e8..f76968f 100644 --- a/network_camera_receiver/include/network_camera_receiver.h +++ b/network_camera_receiver/include/network_camera_receiver.h @@ -13,6 +13,15 @@ #include #include #include +#include +#include + +struct NetworkFrame { + cv::Mat image; + std::uint64_t frame_id = 0; + std::chrono::steady_clock::time_point captured_at; + std::int64_t captured_at_unix_ms = 0; +}; class NetworkCameraReceiver { public: @@ -27,6 +36,7 @@ public: void stop(); // 取最新帧（深拷贝到 out）；自上次取帧后若无新帧则返回 false bool getLatestFrame(cv::Mat& out); + bool getLatestFrame(NetworkFrame& out); // 视频流是否已打开 bool isOpened() const; @@ -40,6 +50,9 @@ private: std::thread capture_thread; std::mutex mtx; // 保护 latest_frame cv::Mat latest_frame; // 最新一帧 + std::uint64_t latest_frame_id = 0; + std::chrono::steady_clock::time_point latest_captured_at; + std::int64_t latest_captured_at_unix_ms = 0; std::atomic has_new_frame{false}; std::atomic running{false}; }; diff --git a/network_camera_receiver/src/network_camera_receiver.cpp b/network_camera_receiver/src/network_camera_receiver.cpp index 39fe617..4a31913 100644 --- a/network_camera_receiver/src/network_camera_receiver.cpp +++ b/network_camera_receiver/src/network_camera_receiver.cpp @@ -57,17 +57,31 @@ void NetworkCameraReceiver::captureLoop() { std::lock_guard lock(mtx); // 仅保留最新一帧，旧帧直接丢弃 frame.copyTo(latest_frame); + ++latest_frame_id; + latest_captured_at = std::chrono::steady_clock::now(); + latest_captured_at_unix_ms = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(); } has_new_frame.store(true); } } bool NetworkCameraReceiver::getLatestFrame(cv::Mat& out) { + NetworkFrame frame; + if (!getLatestFrame(frame)) return false; + out = frame.image; + return true; +} + +bool NetworkCameraReceiver::getLatestFrame(NetworkFrame& out) { if (!has_new_frame.load()) return false; // 先清标记：若在此期间后台又写入新帧，标记会被再次置 true，下一轮可取到 has_new_frame.store(false); std::lock_guard lock(mtx); if (latest_frame.empty()) return false; - latest_frame.copyTo(out); + latest_frame.copyTo(out.image); + out.frame_id = latest_frame_id; + out.captured_at = latest_captured_at; + out.captured_at_unix_ms = latest_captured_at_unix_ms; return true; } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 0000000..8995619 --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,13 @@ +add_executable(face_pipeline_tests + face_pipeline_tests.cpp +) + +target_link_libraries(face_pipeline_tests PRIVATE + face_pipeline +) + +target_compile_definitions(face_pipeline_tests PRIVATE + SOURCE_ROOT="${CMAKE_SOURCE_DIR}" +) + +add_test(NAME face_pipeline_tests COMMAND face_pipeline_tests) diff --git a/tests/face_pipeline_tests.cpp b/tests/face_pipeline_tests.cpp new file mode 100644 index 0000000..6e4ec3b --- /dev/null +++ b/tests/face_pipeline_tests.cpp @@ -0,0 +1,137 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include "face_pipeline/face_aligner.h" +#include "face_pipeline/face_embedder.h" +#include "face_pipeline/iou_tracker.h" +#include "face_pipeline/motion_detector.h" +#include "face_pipeline/result_sink.h" + +namespace { + +using face_pipeline::SteadyTime; + +void require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} + +face_pipeline::FaceObservation observation(int x, int y, int width, int height) { + face_pipeline::FaceObservation face; + face.bbox = cv::Rect(x, y, width, height); + face.score = 0.9f; + return face; +} + +void testMotionDetector() { + face_pipeline::MotionDetector detector; + const SteadyTime start = std::chrono::steady_clock::now(); + const cv::Mat black(180, 320, CV_8UC3, cv::Scalar::all(0)); + cv::Mat moving = black.clone(); + cv::rectangle(moving, cv::Rect(40, 40, 80, 80), cv::Scalar::all(255), -1); + + require(!detector.shouldProcess(black, start, false), "first frame must initialize"); + require(!detector.shouldProcess(black, start + std::chrono::milliseconds(100), false), + "static frame must not trigger"); + require(detector.shouldProcess(moving, start + std::chrono::milliseconds(200), false), + "local movement must trigger"); + require(detector.shouldProcess(moving, start + std::chrono::milliseconds(600), false), + "motion hold must remain active"); + require(detector.shouldProcess(moving, start + std::chrono::milliseconds(1600), true), + "active tracks must force a one-second scan"); +} + +void testFaceAligner() { + face_pipeline::FaceObservation face = observation(20, 20, 80, 80); + face.landmarks = {{{38.2946f, 51.6963f}, {73.5318f, 51.5014f}, + {56.0252f, 71.7366f}, {41.5493f, 92.3655f}, + {70.7299f, 92.2041f}}}; + cv::Mat input(112, 112, CV_8UC3); + for (int row = 0; row < input.rows; ++row) { + for (int col = 0; col < input.cols; ++col) { + input.at(row, col) = cv::Vec3b(row, col, (row + col) / 2); + } + } + + face_pipeline::FaceAligner aligner; + cv::Mat aligned; + require(aligner.align(input, face, aligned), "template landmarks must align"); + require(aligned.size() == cv::Size(112, 112), "aligned size must be 112x112"); + require(cv::norm(input, aligned, cv::NORM_INF) <= 2.0, "template alignment must be identity"); + + face.bbox = cv::Rect(0, 0, 5, 5); + require(!aligner.align(input, face, aligned), "tiny faces must be rejected"); +} + +void testIouTracker() { + face_pipeline::IouTracker tracker; + const SteadyTime start = std::chrono::steady_clock::now(); + + std::vector first = tracker.update( + {observation(10, 10, 50, 50), observation(200, 10, 50, 50)}, start); + require(first.size() == 2 && first[0].is_new && first[1].is_new, + "first observations must create tracks"); + const std::uint64_t first_id = first[0].track_id; + + std::vector matched = tracker.update( + {observation(14, 12, 50, 50)}, start + std::chrono::milliseconds(500)); + require(matched.size() == 1 && matched[0].track_id == first_id && !matched[0].is_new, + "overlapping observation must preserve track id"); + + std::vector expired = tracker.update( + {observation(14, 12, 50, 50)}, start + std::chrono::milliseconds(2600)); + require(expired.size() == 1 && expired[0].track_id != first_id && expired[0].is_new, + "expired observation must receive a new track id"); +} + +void testLoggingSink() { + std::ostringstream output; + face_pipeline::LoggingResultSink sink(output); + face_pipeline::FaceFeatureEvent event; + event.track_id = 7; + event.frame_id = 42; + event.embedding_model = "test_model"; + event.embedding.assign(128, 0.0f); + require(sink.publish(event), "logging sink must report success"); + require(output.str().find("track_id=7") != std::string::npos, + "logging sink must include the track id"); + require(output.str().find("embedding_dim=128") != std::string::npos, + "logging sink must include embedding dimension"); +} + +void testSFaceModel() { + face_pipeline::SFaceEmbedder embedder( + std::string(SOURCE_ROOT) + "/models/face_recognition_sface_2021dec.onnx"); + cv::Mat input(112, 112, CV_8UC3, cv::Scalar(90, 120, 150)); + const std::vector embedding = embedder.extract(input); + require(embedding.size() == 128, "SFace must return 128 values"); + double squared_norm = 0.0; + for (float value : embedding) { + require(std::isfinite(value), "SFace values must be finite"); + squared_norm += static_cast(value) * value; + } + require(std::abs(std::sqrt(squared_norm) - 1.0) < 1e-5, + "SFace embedding must be L2 normalized"); +} + +} // namespace + +int main() { + try { + testMotionDetector(); + testFaceAligner(); + testIouTracker(); + testLoggingSink(); + testSFaceModel(); + } catch (const std::exception& error) { + std::cerr << "face_pipeline_tests failed: " << error.what() << std::endl; + return 1; + } + std::cout << "face_pipeline_tests passed" << std::endl; + return 0; +}