Add files

PyPatchMatch/.gitignore

@@ -0,0 +1,4 @@
+/build/
+/*.so
+__pycache__
+*.py[cod]

PyPatchMatch/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Jiayuan Mao
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

PyPatchMatch/Makefile

@@ -0,0 +1,54 @@
+#
+# Makefile
+# Jiayuan Mao, 2019-01-09 13:59
+#
+
+SRC_DIR = csrc
+INC_DIR = csrc
+OBJ_DIR = build/obj
+TARGET = libpatchmatch.so
+
+LIB_TARGET = $(TARGET)
+INCLUDE_DIR = -I $(SRC_DIR) -I $(INC_DIR)
+
+CXX = $(ENVIRONMENT_OPTIONS) g++
+CXXFLAGS = -std=c++14
+CXXFLAGS += -Ofast -ffast-math -w
+# CXXFLAGS += -g
+CXXFLAGS += $(shell pkg-config --cflags opencv) -fPIC
+CXXFLAGS += $(INCLUDE_DIR)
+LDFLAGS = $(shell pkg-config --cflags --libs opencv) -shared -fPIC
+
+
+CXXSOURCES = $(shell find $(SRC_DIR)/ -name "*.cpp")
+OBJS = $(addprefix $(OBJ_DIR)/,$(CXXSOURCES:.cpp=.o))
+DEPFILES = $(OBJS:.o=.d)
+
+.PHONY: all clean rebuild test
+
+all: $(LIB_TARGET)
+
+$(OBJ_DIR)/%.o: %.cpp
+	@echo "[CC] $< ..."
+	@$(CXX) -c $< $(CXXFLAGS) -o $@
+
+$(OBJ_DIR)/%.d: %.cpp
+	@mkdir -pv $(dir $@)
+	@echo "[dep] $< ..."
+	@$(CXX) $(INCLUDE_DIR) $(CXXFLAGS) -MM -MT "$(OBJ_DIR)/$(<:.cpp=.o) $(OBJ_DIR)/$(<:.cpp=.d)" "$<" > "$@"
+
+sinclude $(DEPFILES)
+
+$(LIB_TARGET): $(OBJS)
+	@echo "[link] $(LIB_TARGET) ..."
+	@$(CXX) $(OBJS) -o $@ $(CXXFLAGS) $(LDFLAGS)
+
+clean:
+	rm -rf $(OBJ_DIR) $(LIB_TARGET)
+
+rebuild:
+	+@make clean
+	+@make
+
+# vim:ft=make
+#

PyPatchMatch/README.md

@@ -0,0 +1,64 @@
+PatchMatch based Inpainting
+=====================================
+This library implements the PatchMatch based inpainting algorithm. It provides both C++ and Python interfaces.
+This implementation is heavily based on the implementation by Younesse ANDAM:
+(younesse-cv/PatchMatch)[https://github.com/younesse-cv/PatchMatch], with some bugs fix.
+
+Usage
+-------------------------------------
+
+You need to first install OpenCV to compile the C++ libraries. Then, run `make` to compile the
+shared library `libpatchmatch.so`.
+
+For Python users (example available at `examples/py_example.py`)
+
+```python
+import patch_match
+
+image = ...  # either a numpy ndarray or a PIL Image object.
+mask = ...   # either a numpy ndarray or a PIL Image object.
+result = patch_match.inpaint(image, mask, patch_size=5)
+```
+
+For C++ users (examples available at `examples/cpp_example.cpp`)
+
+```cpp
+#include "inpaint.h"
+
+int main() {
+    cv::Mat image = ...
+    cv::Mat mask = ...
+
+    cv::Mat result = Inpainting(image, mask, 5).run();
+
+    return 0;
+}
+```
+
+
+README and COPYRIGHT by Younesse ANDAM
+-------------------------------------
+@Author: Younesse ANDAM
+
+@Contact: younesse.andam@gmail.com
+
+Description: This project is a personal implementation of an algorithm called PATCHMATCH that restores missing areas in an image.
+The algorithm is presented in the following paper
+ PatchMatch  A Randomized Correspondence Algorithm
+               for Structural Image Editing
+   by C.Barnes,E.Shechtman,A.Finkelstein and Dan B.Goldman
+   ACM Transactions on Graphics (Proc. SIGGRAPH), vol.28, aug-2009
+
+ For more information please refer to
+ http://www.cs.princeton.edu/gfx/pubs/Barnes_2009_PAR/index.php
+
+Copyright (c) 2010-2011
+
+
+Requirements
+-------------------------------------
+
+To run the project you need to install Opencv library and link it to your project.
+Opencv can be download it here
+http://opencv.org/downloads.html
+

PyPatchMatch/csrc/inpaint.cpp

@@ -0,0 +1,234 @@
+#include <algorithm>
+#include <iostream>
+#include <opencv2/imgcodecs.hpp>
+#include <opencv2/imgproc.hpp>
+#include <opencv2/highgui.hpp>
+
+#include "inpaint.h"
+
+namespace {
+    static std::vector<double> kDistance2Similarity;
+
+    void init_kDistance2Similarity() {
+        double base[11] = {1.0, 0.99, 0.96, 0.83, 0.38, 0.11, 0.02, 0.005, 0.0006, 0.0001, 0};
+        int length = (PatchDistanceMetric::kDistanceScale + 1);
+        kDistance2Similarity.resize(length);
+        for (int i = 0; i < length; ++i) {
+            double t = (double) i / length;
+            int j = (int) (100 * t);
+            int k = j + 1;
+            double vj = (j < 11) ? base[j] : 0;
+            double vk = (k < 11) ? base[k] : 0;
+            kDistance2Similarity[i] = vj + (100 * t - j) * (vk - vj);
+        }
+    }
+
+
+    inline void _weighted_copy(const MaskedImage &source, int ys, int xs, cv::Mat &target, int yt, int xt, double weight) {
+        if (source.is_masked(ys, xs)) return;
+        if (source.is_globally_masked(ys, xs)) return;
+
+        auto source_ptr = source.get_image(ys, xs);
+        auto target_ptr = target.ptr<double>(yt, xt);
+
+#pragma unroll
+        for (int c = 0; c < 3; ++c)
+            target_ptr[c] += static_cast<double>(source_ptr[c]) * weight;
+        target_ptr[3] += weight;
+    }
+}
+
+/**
+ * This algorithme uses a version proposed by Xavier Philippeau.
+ */
+
+Inpainting::Inpainting(cv::Mat image, cv::Mat mask, const PatchDistanceMetric *metric)
+    : m_initial(image, mask), m_distance_metric(metric), m_pyramid(), m_source2target(), m_target2source() {
+    _initialize_pyramid();
+}
+
+Inpainting::Inpainting(cv::Mat image, cv::Mat mask, cv::Mat global_mask, const PatchDistanceMetric *metric)
+    : m_initial(image, mask, global_mask), m_distance_metric(metric), m_pyramid(), m_source2target(), m_target2source() {
+    _initialize_pyramid();
+}
+
+void Inpainting::_initialize_pyramid() {
+    auto source = m_initial;
+    m_pyramid.push_back(source);
+    while (source.size().height > m_distance_metric->patch_size() && source.size().width > m_distance_metric->patch_size()) {
+        source = source.downsample();
+        m_pyramid.push_back(source);
+    }
+
+    if (kDistance2Similarity.size() == 0) {
+        init_kDistance2Similarity();
+    }
+}
+
+cv::Mat Inpainting::run(bool verbose, bool verbose_visualize, unsigned int random_seed) {
+    srand(random_seed);
+    const int nr_levels = m_pyramid.size();
+
+    MaskedImage source, target;
+    for (int level = nr_levels - 1; level >= 0; --level) {
+        if (verbose) std::cerr << "Inpainting level: " << level << std::endl;
+
+        source = m_pyramid[level];
+
+        if (level == nr_levels - 1) {
+            target = source.clone();
+            target.clear_mask();
+            m_source2target = NearestNeighborField(source, target, m_distance_metric);
+            m_target2source = NearestNeighborField(target, source, m_distance_metric);
+        } else {
+            m_source2target = NearestNeighborField(source, target, m_distance_metric, m_source2target);
+            m_target2source = NearestNeighborField(target, source, m_distance_metric, m_target2source);
+        }
+
+        if (verbose) std::cerr << "Initialization done." << std::endl;
+
+        if (verbose_visualize) {
+            auto visualize_size = m_initial.size();
+            cv::Mat source_visualize(visualize_size, m_initial.image().type());
+            cv::resize(source.image(), source_visualize, visualize_size);
+            cv::imshow("Source", source_visualize);
+            cv::Mat target_visualize(visualize_size, m_initial.image().type());
+            cv::resize(target.image(), target_visualize, visualize_size);
+            cv::imshow("Target", target_visualize);
+            cv::waitKey(0);
+        }
+
+        target = _expectation_maximization(source, target, level, verbose);
+    }
+
+    return target.image();
+}
+
+// EM-Like algorithm (see "PatchMatch" - page 6).
+// Returns a double sized target image (unless level = 0).
+MaskedImage Inpainting::_expectation_maximization(MaskedImage source, MaskedImage target, int level, bool verbose) {
+    const int nr_iters_em = 1 + 2 * level;
+    const int nr_iters_nnf = static_cast<int>(std::min(7, 1 + level));
+    const int patch_size = m_distance_metric->patch_size();
+
+    MaskedImage new_source, new_target;
+
+    for (int iter_em = 0; iter_em < nr_iters_em; ++iter_em) {
+        if (iter_em != 0) {
+            m_source2target.set_target(new_target);
+            m_target2source.set_source(new_target);
+            target = new_target;
+        }
+
+        if (verbose) std::cerr << "EM Iteration: " << iter_em << std::endl;
+
+        auto size = source.size();
+        for (int i = 0; i < size.height; ++i) {
+            for (int j = 0; j < size.width; ++j) {
+                if (!source.contains_mask(i, j, patch_size)) {
+                    m_source2target.set_identity(i, j);
+                    m_target2source.set_identity(i, j);
+                }
+            }
+        }
+        if (verbose) std::cerr << "  NNF minimization started." << std::endl;
+        m_source2target.minimize(nr_iters_nnf);
+        m_target2source.minimize(nr_iters_nnf);
+        if (verbose) std::cerr << "  NNF minimization finished." << std::endl;
+
+        // Instead of upsizing the final target, we build the last target from the next level source image.
+        // Thus, the final target is less blurry (see "Space-Time Video Completion" - page 5).
+        bool upscaled = false;
+        if (level >= 1 && iter_em == nr_iters_em - 1) {
+            new_source = m_pyramid[level - 1];
+            new_target = target.upsample(new_source.size().width, new_source.size().height, m_pyramid[level - 1].global_mask());
+            upscaled = true;
+        } else {
+            new_source = m_pyramid[level];
+            new_target = target.clone();
+        }
+
+        auto vote = cv::Mat(new_target.size(), CV_64FC4);
+        vote.setTo(cv::Scalar::all(0));
+
+        // Votes for best patch from NNF Source->Target (completeness) and Target->Source (coherence).
+        _expectation_step(m_source2target, 1, vote, new_source, upscaled);
+        if (verbose) std::cerr << "  Expectation source to target finished." << std::endl;
+        _expectation_step(m_target2source, 0, vote, new_source, upscaled);
+        if (verbose) std::cerr << "  Expectation target to source finished." << std::endl;
+
+        // Compile votes and update pixel values.
+        _maximization_step(new_target, vote);
+        if (verbose) std::cerr << "  Minimization step finished." << std::endl;
+    }
+
+    return new_target;
+}
+
+// Expectation step: vote for best estimations of each pixel.
+void Inpainting::_expectation_step(
+    const NearestNeighborField &nnf, bool source2target,
+    cv::Mat &vote, const MaskedImage &source, bool upscaled
+) {
+    auto source_size = nnf.source_size();
+    auto target_size = nnf.target_size();
+    const int patch_size = m_distance_metric->patch_size();
+
+    for (int i = 0; i < source_size.height; ++i) {
+        for (int j = 0; j < source_size.width; ++j) {
+            if (nnf.source().is_globally_masked(i, j)) continue;
+            int yp = nnf.at(i, j, 0), xp = nnf.at(i, j, 1), dp = nnf.at(i, j, 2);
+            double w = kDistance2Similarity[dp];
+
+            for (int di = -patch_size; di <= patch_size; ++di) {
+                for (int dj = -patch_size; dj <= patch_size; ++dj) {
+                    int ys = i + di, xs = j + dj, yt = yp + di, xt = xp + dj;
+                    if (!(ys >= 0 && ys < source_size.height && xs >= 0 && xs < source_size.width)) continue;
+                    if (nnf.source().is_globally_masked(ys, xs)) continue;
+                    if (!(yt >= 0 && yt < target_size.height && xt >= 0 && xt < target_size.width)) continue;
+                    if (nnf.target().is_globally_masked(yt, xt)) continue;
+
+                    if (!source2target) {
+                        std::swap(ys, yt);
+                        std::swap(xs, xt);
+                    }
+
+                    if (upscaled) {
+                        for (int uy = 0; uy < 2; ++uy) {
+                            for (int ux = 0; ux < 2; ++ux) {
+                                _weighted_copy(source, 2 * ys + uy, 2 * xs + ux, vote, 2 * yt + uy, 2 * xt + ux, w);
+                            }
+                        }
+                    } else {
+                        _weighted_copy(source, ys, xs, vote, yt, xt, w);
+                    }
+                }
+            }
+        }
+    }
+}
+
+// Maximization Step: maximum likelihood of target pixel.
+void Inpainting::_maximization_step(MaskedImage &target, const cv::Mat &vote) {
+    auto target_size = target.size();
+    for (int i = 0; i < target_size.height; ++i) {
+        for (int j = 0; j < target_size.width; ++j) {
+            const double *source_ptr = vote.ptr<double>(i, j);
+            unsigned char *target_ptr = target.get_mutable_image(i, j);
+
+            if (target.is_globally_masked(i, j)) {
+                continue;
+            }
+
+            if (source_ptr[3] > 0) {
+                unsigned char r = cv::saturate_cast<unsigned char>(source_ptr[0] / source_ptr[3]);
+                unsigned char g = cv::saturate_cast<unsigned char>(source_ptr[1] / source_ptr[3]);
+                unsigned char b = cv::saturate_cast<unsigned char>(source_ptr[2] / source_ptr[3]);
+                target_ptr[0] = r, target_ptr[1] = g, target_ptr[2] = b;
+            } else {
+                target.set_mask(i, j, 0);
+            }
+        }
+    }
+}
+

PyPatchMatch/csrc/inpaint.h

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <vector>
+
+#include "masked_image.h"
+#include "nnf.h"
+
+class Inpainting {
+public:
+    Inpainting(cv::Mat image, cv::Mat mask, const PatchDistanceMetric *metric);
+    Inpainting(cv::Mat image, cv::Mat mask, cv::Mat global_mask, const PatchDistanceMetric *metric);
+    cv::Mat run(bool verbose = false, bool verbose_visualize = false, unsigned int random_seed = 1212);
+
+private:
+    void _initialize_pyramid(void);
+    MaskedImage _expectation_maximization(MaskedImage source, MaskedImage target, int level, bool verbose);
+    void _expectation_step(const NearestNeighborField &nnf, bool source2target, cv::Mat &vote, const MaskedImage &source, bool upscaled);
+    void _maximization_step(MaskedImage &target, const cv::Mat &vote);
+
+    MaskedImage m_initial;
+    std::vector<MaskedImage> m_pyramid;
+
+    NearestNeighborField m_source2target;
+    NearestNeighborField m_target2source;
+    const PatchDistanceMetric *m_distance_metric;
+};
+

PyPatchMatch/csrc/masked_image.cpp

@@ -0,0 +1,138 @@
+#include "masked_image.h"
+#include <algorithm>
+#include <iostream>
+
+const cv::Size MaskedImage::kDownsampleKernelSize = cv::Size(6, 6);
+const int MaskedImage::kDownsampleKernel[6] = {1, 5, 10, 10, 5, 1};
+
+bool MaskedImage::contains_mask(int y, int x, int patch_size) const {
+    auto mask_size = size();
+    for (int dy = -patch_size; dy <= patch_size; ++dy) {
+        for (int dx = -patch_size; dx <= patch_size; ++dx) {
+            int yy = y + dy, xx = x + dx;
+            if (yy >= 0 && yy < mask_size.height && xx >= 0 && xx < mask_size.width) {
+                if (is_masked(yy, xx) && !is_globally_masked(yy, xx)) return true;
+            }
+        }
+    }
+    return false;
+}
+
+MaskedImage MaskedImage::downsample() const {
+    const auto &kernel_size = MaskedImage::kDownsampleKernelSize;
+    const auto &kernel = MaskedImage::kDownsampleKernel;
+
+    const auto size = this->size();
+    const auto new_size = cv::Size(size.width / 2, size.height / 2);
+
+    auto ret = MaskedImage(new_size.width, new_size.height);
+    if (!m_global_mask.empty()) ret.init_global_mask_mat();
+    for (int y = 0; y < size.height - 1; y += 2) {
+        for (int x = 0; x < size.width - 1; x += 2) {
+            int r = 0, g = 0, b = 0, ksum = 0;
+            bool is_gmasked = true;
+
+            for (int dy = -kernel_size.height / 2 + 1; dy <= kernel_size.height / 2; ++dy) {
+                for (int dx = -kernel_size.width / 2 + 1; dx <= kernel_size.width / 2; ++dx) {
+                    int yy = y + dy, xx = x + dx;
+                    if (yy >= 0 && yy < size.height && xx >= 0 && xx < size.width) {
+                        if (!is_globally_masked(yy, xx)) {
+                            is_gmasked = false;
+                        }
+                        if (!is_masked(yy, xx)) {
+                            auto source_ptr = get_image(yy, xx);
+                            int k = kernel[kernel_size.height / 2 - 1 + dy] * kernel[kernel_size.width / 2 - 1 + dx];
+                            r += source_ptr[0] * k, g += source_ptr[1] * k, b += source_ptr[2] * k;
+                            ksum += k;
+                        }
+                    }
+                }
+            }
+
+            if (ksum > 0) r /= ksum, g /= ksum, b /= ksum;
+
+            if (!m_global_mask.empty()) {
+                ret.set_global_mask(y / 2, x / 2, is_gmasked);
+            }
+            if (ksum > 0) {
+                auto target_ptr = ret.get_mutable_image(y / 2, x / 2);
+                target_ptr[0] = r, target_ptr[1] = g, target_ptr[2] = b;
+                ret.set_mask(y / 2, x / 2, 0);
+            } else {
+                ret.set_mask(y / 2, x / 2, 1);
+            }
+        }
+    }
+
+    return ret;
+}
+
+MaskedImage MaskedImage::upsample(int new_w, int new_h) const {
+    const auto size = this->size();
+    auto ret = MaskedImage(new_w, new_h);
+    if (!m_global_mask.empty()) ret.init_global_mask_mat();
+    for (int y = 0; y < new_h; ++y) {
+        for (int x = 0; x < new_w; ++x) {
+            int yy = y * size.height / new_h;
+            int xx = x * size.width / new_w;
+
+            if (is_globally_masked(yy, xx)) {
+                ret.set_global_mask(y, x, 1);
+                ret.set_mask(y, x, 1);
+            } else {
+                if (!m_global_mask.empty()) ret.set_global_mask(y, x, 0);
+
+                if (is_masked(yy, xx)) {
+                    ret.set_mask(y, x, 1);
+                } else {
+                    auto source_ptr = get_image(yy, xx);
+                    auto target_ptr = ret.get_mutable_image(y, x);
+                    for (int c = 0; c < 3; ++c)
+                        target_ptr[c] = source_ptr[c];
+                    ret.set_mask(y, x, 0);
+                }
+            }
+        }
+    }
+
+    return ret;
+}
+
+MaskedImage MaskedImage::upsample(int new_w, int new_h, const cv::Mat &new_global_mask) const {
+    auto ret = upsample(new_w, new_h);
+    ret.set_global_mask_mat(new_global_mask);
+    return ret;
+}
+
+void MaskedImage::compute_image_gradients() {
+    if (m_image_grad_computed) {
+        return;
+    }
+
+    const auto size = m_image.size();
+    m_image_grady = cv::Mat(size, CV_8UC3);
+    m_image_gradx = cv::Mat(size, CV_8UC3);
+    m_image_grady = cv::Scalar::all(0);
+    m_image_gradx = cv::Scalar::all(0);
+
+    for (int i = 1; i < size.height - 1; ++i) {
+        const auto *ptr = m_image.ptr<unsigned char>(i, 0);
+        const auto *ptry1 = m_image.ptr<unsigned char>(i + 1, 0);
+        const auto *ptry2 = m_image.ptr<unsigned char>(i - 1, 0);
+        const auto *ptrx1 = m_image.ptr<unsigned char>(i, 0) + 3;
+        const auto *ptrx2 = m_image.ptr<unsigned char>(i, 0) - 3;
+        auto *mptry = m_image_grady.ptr<unsigned char>(i, 0);
+        auto *mptrx = m_image_gradx.ptr<unsigned char>(i, 0);
+        for (int j = 3; j < size.width * 3 - 3; ++j) {
+            mptry[j] = (ptry1[j] / 2 - ptry2[j] / 2) + 128;
+            mptrx[j] = (ptrx1[j] / 2 - ptrx2[j] / 2) + 128;
+        }
+    }
+
+    m_image_grad_computed = true;
+}
+
+void MaskedImage::compute_image_gradients() const {
+    const_cast<MaskedImage *>(this)->compute_image_gradients();
+}
+

PyPatchMatch/csrc/masked_image.h

@@ -0,0 +1,112 @@
+#pragma once
+
+#include <opencv2/core.hpp>
+
+class MaskedImage {
+public:
+    MaskedImage() : m_image(), m_mask(), m_global_mask(), m_image_grady(), m_image_gradx(), m_image_grad_computed(false) {
+        // pass
+    }
+    MaskedImage(cv::Mat image, cv::Mat mask) : m_image(image), m_mask(mask), m_image_grad_computed(false) {
+        // pass
+    }
+    MaskedImage(cv::Mat image, cv::Mat mask, cv::Mat global_mask) : m_image(image), m_mask(mask), m_global_mask(global_mask), m_image_grad_computed(false) {
+        // pass
+    }
+    MaskedImage(cv::Mat image, cv::Mat mask, cv::Mat global_mask, cv::Mat grady, cv::Mat gradx, bool grad_computed) :
+        m_image(image), m_mask(mask), m_global_mask(global_mask),
+        m_image_grady(grady), m_image_gradx(gradx), m_image_grad_computed(grad_computed) {
+        // pass
+    }
+    MaskedImage(int width, int height) : m_global_mask(), m_image_grady(), m_image_gradx() {
+        m_image = cv::Mat(cv::Size(width, height), CV_8UC3);
+        m_image = cv::Scalar::all(0);
+
+        m_mask = cv::Mat(cv::Size(width, height), CV_8U);
+        m_mask = cv::Scalar::all(0);
+    }
+    inline MaskedImage clone() {
+        return MaskedImage(
+            m_image.clone(), m_mask.clone(), m_global_mask.clone(),
+            m_image_grady.clone(), m_image_gradx.clone(), m_image_grad_computed
+        );
+    }
+
+    inline cv::Size size() const {
+        return m_image.size();
+    }
+    inline const cv::Mat &image() const {
+        return m_image;
+    }
+    inline const cv::Mat &mask() const {
+        return m_mask;
+    }
+    inline const cv::Mat &global_mask() const {
+        return m_global_mask;
+    }
+    inline const cv::Mat &grady() const {
+        assert(m_image_grad_computed);
+        return m_image_grady;
+    }
+    inline const cv::Mat &gradx() const {
+        assert(m_image_grad_computed);
+        return m_image_gradx;
+    }
+
+    inline void init_global_mask_mat() {
+        m_global_mask = cv::Mat(m_mask.size(), CV_8U);
+        m_global_mask.setTo(cv::Scalar(0));
+    }
+    inline void set_global_mask_mat(const cv::Mat &other) {
+        m_global_mask = other;
+    }
+
+    inline bool is_masked(int y, int x) const {
+        return static_cast<bool>(m_mask.at<unsigned char>(y, x));
+    }
+    inline bool is_globally_masked(int y, int x) const {
+        return !m_global_mask.empty() && static_cast<bool>(m_global_mask.at<unsigned char>(y, x));
+    }
+    inline void set_mask(int y, int x, bool value) {
+        m_mask.at<unsigned char>(y, x) = static_cast<unsigned char>(value);
+    }
+    inline void set_global_mask(int y, int x, bool value) {
+        m_global_mask.at<unsigned char>(y, x) = static_cast<unsigned char>(value);
+    }
+    inline void clear_mask() {
+        m_mask.setTo(cv::Scalar(0));
+    }
+
+    inline const unsigned char *get_image(int y, int x) const {
+        return m_image.ptr<unsigned char>(y, x);
+    }
+    inline unsigned char *get_mutable_image(int y, int x) {
+        return m_image.ptr<unsigned char>(y, x);
+    }
+
+    inline unsigned char get_image(int y, int x, int c) const {
+        return m_image.ptr<unsigned char>(y, x)[c];
+    }
+    inline int get_image_int(int y, int x, int c) const {
+        return static_cast<int>(m_image.ptr<unsigned char>(y, x)[c]);
+    }
+
+    bool contains_mask(int y, int x, int patch_size) const;
+    MaskedImage downsample() const;
+    MaskedImage upsample(int new_w, int new_h) const;
+    MaskedImage upsample(int new_w, int new_h, const cv::Mat &new_global_mask) const;
+    void compute_image_gradients();
+    void compute_image_gradients() const;
+
+    static const cv::Size kDownsampleKernelSize;
+    static const int kDownsampleKernel[6];
+
+private:
+	cv::Mat m_image;
+	cv::Mat m_mask;
+    cv::Mat m_global_mask;
+    cv::Mat m_image_grady;
+    cv::Mat m_image_gradx;
+    bool m_image_grad_computed = false;
+};
+

PyPatchMatch/csrc/nnf.cpp

@@ -0,0 +1,268 @@
+#include <algorithm>
+#include <iostream>
+#include <cmath>
+
+#include "masked_image.h"
+#include "nnf.h"
+
+/**
+* Nearest-Neighbor Field (see PatchMatch algorithm).
+* This algorithme uses a version proposed by Xavier Philippeau.
+*
+*/
+
+template <typename T>
+T clamp(T value, T min_value, T max_value) {
+    return std::min(std::max(value, min_value), max_value);
+}
+
+void NearestNeighborField::_randomize_field(int max_retry, bool reset) {
+    auto this_size = source_size();
+    for (int i = 0; i < this_size.height; ++i) {
+        for (int j = 0; j < this_size.width; ++j) {
+            if (m_source.is_globally_masked(i, j)) continue;
+
+            auto this_ptr = mutable_ptr(i, j);
+            int distance = reset ? PatchDistanceMetric::kDistanceScale : this_ptr[2];
+            if (distance < PatchDistanceMetric::kDistanceScale) {
+                continue;
+            }
+
+            int i_target = 0, j_target = 0;
+            for (int t = 0; t < max_retry; ++t) {
+                i_target = rand() % this_size.height;
+                j_target = rand() % this_size.width;
+                if (m_target.is_globally_masked(i_target, j_target)) continue;
+
+                distance = _distance(i, j, i_target, j_target);
+                if (distance < PatchDistanceMetric::kDistanceScale)
+                    break;
+            }
+
+            this_ptr[0] = i_target, this_ptr[1] = j_target, this_ptr[2] = distance;
+        }
+    }
+}
+
+void NearestNeighborField::_initialize_field_from(const NearestNeighborField &other, int max_retry) {
+    const auto &this_size = source_size();
+    const auto &other_size = other.source_size();
+    double fi = static_cast<double>(this_size.height) / other_size.height;
+    double fj = static_cast<double>(this_size.width) / other_size.width;
+
+    for (int i = 0; i < this_size.height; ++i) {
+        for (int j = 0; j < this_size.width; ++j) {
+            if (m_source.is_globally_masked(i, j)) continue;
+
+            int ilow = static_cast<int>(std::min(i / fi, static_cast<double>(other_size.height - 1)));
+            int jlow = static_cast<int>(std::min(j / fj, static_cast<double>(other_size.width - 1)));
+            auto this_value = mutable_ptr(i, j);
+            auto other_value = other.ptr(ilow, jlow);
+
+            this_value[0] = static_cast<int>(other_value[0] * fi);
+            this_value[1] = static_cast<int>(other_value[1] * fj);
+            this_value[2] = _distance(i, j, this_value[0], this_value[1]);
+        }
+    }
+
+    _randomize_field(max_retry, false);
+}
+
+void NearestNeighborField::minimize(int nr_pass) {
+    const auto &this_size = source_size();
+    while (nr_pass--) {
+        for (int i = 0; i < this_size.height; ++i)
+            for (int j = 0; j < this_size.width; ++j) {
+                if (m_source.is_globally_masked(i, j)) continue;
+                if (at(i, j, 2) > 0) _minimize_link(i, j, +1);
+            }
+        for (int i = this_size.height - 1; i >= 0; --i)
+            for (int j = this_size.width - 1; j >= 0; --j) {
+                if (m_source.is_globally_masked(i, j)) continue;
+                if (at(i, j, 2) > 0) _minimize_link(i, j, -1);
+            }
+    }
+}
+
+void NearestNeighborField::_minimize_link(int y, int x, int direction) {
+    const auto &this_size = source_size();
+    const auto &this_target_size = target_size();
+    auto this_ptr = mutable_ptr(y, x);
+
+    // propagation along the y direction.
+    if (y - direction >= 0 && y - direction < this_size.height && !m_source.is_globally_masked(y - direction, x)) {
+        int yp = at(y - direction, x, 0) + direction;
+        int xp = at(y - direction, x, 1);
+        int dp = _distance(y, x, yp, xp);
+        if (dp < at(y, x, 2)) {
+            this_ptr[0] = yp, this_ptr[1] = xp, this_ptr[2] = dp;
+        }
+    }
+
+    // propagation along the x direction.
+    if (x - direction >= 0 && x - direction < this_size.width && !m_source.is_globally_masked(y, x - direction)) {
+        int yp = at(y, x - direction, 0);
+        int xp = at(y, x - direction, 1) + direction;
+        int dp = _distance(y, x, yp, xp);
+        if (dp < at(y, x, 2)) {
+            this_ptr[0] = yp, this_ptr[1] = xp, this_ptr[2] = dp;
+        }
+    }
+
+    // random search with a progressive step size.
+    int random_scale = (std::min(this_target_size.height, this_target_size.width) - 1) / 2;
+    while (random_scale > 0) {
+        int yp = this_ptr[0] + (rand() % (2 * random_scale + 1) - random_scale);
+        int xp = this_ptr[1] + (rand() % (2 * random_scale + 1) - random_scale);
+        yp = clamp(yp, 0, target_size().height - 1);
+        xp = clamp(xp, 0, target_size().width - 1);
+
+        if (m_target.is_globally_masked(yp, xp)) {
+            random_scale /= 2;
+        }
+
+        int dp = _distance(y, x, yp, xp);
+        if (dp < at(y, x, 2)) {
+            this_ptr[0] = yp, this_ptr[1] = xp, this_ptr[2] = dp;
+        }
+        random_scale /= 2;
+    }
+}
+
+const int PatchDistanceMetric::kDistanceScale = 65535;
+const int PatchSSDDistanceMetric::kSSDScale = 9 * 255 * 255;
+
+namespace {
+
+inline int pow2(int i) {
+    return i * i;
+}
+
+int distance_masked_images(
+    const MaskedImage &source, int ys, int xs,
+    const MaskedImage &target, int yt, int xt,
+    int patch_size
+) {
+    long double distance = 0;
+    long double wsum = 0;
+
+    source.compute_image_gradients();
+    target.compute_image_gradients();
+
+    auto source_size = source.size();
+    auto target_size = target.size();
+
+    for (int dy = -patch_size; dy <= patch_size; ++dy) {
+        const int yys = ys + dy, yyt = yt + dy;
+
+        if (yys <= 0 || yys >= source_size.height - 1 || yyt <= 0 || yyt >= target_size.height - 1) {
+            distance += (long double)(PatchSSDDistanceMetric::kSSDScale) * (2 * patch_size + 1);
+            wsum += 2 * patch_size + 1;
+            continue;
+        }
+
+        const auto *p_si = source.image().ptr<unsigned char>(yys, 0);
+        const auto *p_ti = target.image().ptr<unsigned char>(yyt, 0);
+        const auto *p_sm = source.mask().ptr<unsigned char>(yys, 0);
+        const auto *p_tm = target.mask().ptr<unsigned char>(yyt, 0);
+
+        const unsigned char *p_sgm = nullptr;
+        const unsigned char *p_tgm = nullptr;
+        if (!source.global_mask().empty()) {
+            p_sgm = source.global_mask().ptr<unsigned char>(yys, 0);
+            p_tgm = target.global_mask().ptr<unsigned char>(yyt, 0);
+        }
+
+        const auto *p_sgy = source.grady().ptr<unsigned char>(yys, 0);
+        const auto *p_tgy = target.grady().ptr<unsigned char>(yyt, 0);
+        const auto *p_sgx = source.gradx().ptr<unsigned char>(yys, 0);
+        const auto *p_tgx = target.gradx().ptr<unsigned char>(yyt, 0);
+
+        for (int dx = -patch_size; dx <= patch_size; ++dx) {
+            int xxs = xs + dx, xxt = xt + dx;
+            wsum += 1;
+
+            if (xxs <= 0 || xxs >= source_size.width - 1 || xxt <= 0 || xxt >= source_size.width - 1) {
+                distance += PatchSSDDistanceMetric::kSSDScale;
+                continue;
+            }
+
+            if (p_sm[xxs] || p_tm[xxt] || (p_sgm && p_sgm[xxs]) || (p_tgm && p_tgm[xxt]) ) {
+                distance += PatchSSDDistanceMetric::kSSDScale;
+                continue;
+            }
+
+            int ssd = 0;
+            for (int c = 0; c < 3; ++c) {
+                int s_value = p_si[xxs * 3 + c];
+                int t_value = p_ti[xxt * 3 + c];
+                int s_gy = p_sgy[xxs * 3 + c];
+                int t_gy = p_tgy[xxt * 3 + c];
+                int s_gx = p_sgx[xxs * 3 + c];
+                int t_gx = p_tgx[xxt * 3 + c];
+
+                ssd += pow2(static_cast<int>(s_value) - t_value);
+                ssd += pow2(static_cast<int>(s_gx) - t_gx);
+                ssd += pow2(static_cast<int>(s_gy) - t_gy);
+            }
+            distance += ssd;
+        }
+    }
+
+    distance /= (long double)(PatchSSDDistanceMetric::kSSDScale);
+
+    int res = int(PatchDistanceMetric::kDistanceScale * distance / wsum);
+    if (res < 0 || res > PatchDistanceMetric::kDistanceScale) return PatchDistanceMetric::kDistanceScale;
+    return res;
+}
+
+}
+
+int PatchSSDDistanceMetric::operator ()(const MaskedImage &source, int source_y, int source_x, const MaskedImage &target, int target_y, int target_x) const {
+    return distance_masked_images(source, source_y, source_x, target, target_y, target_x, m_patch_size);
+}
+
+int DebugPatchSSDDistanceMetric::operator ()(const MaskedImage &source, int source_y, int source_x, const MaskedImage &target, int target_y, int target_x) const {
+    fprintf(stderr, "DebugPatchSSDDistanceMetric: %d %d %d %d\n", source.size().width, source.size().height, m_width, m_height);
+    return distance_masked_images(source, source_y, source_x, target, target_y, target_x, m_patch_size);
+}
+
+int RegularityGuidedPatchDistanceMetricV1::operator ()(const MaskedImage &source, int source_y, int source_x, const MaskedImage &target, int target_y, int target_x) const {
+    double dx = remainder(double(source_x - target_x) / source.size().width, m_dx1);
+    double dy = remainder(double(source_y - target_y) / source.size().height, m_dy2);
+
+    double score1 = sqrt(dx * dx + dy *dy) / m_scale;
+    if (score1 < 0 || score1 > 1) score1 = 1;
+    score1 *= PatchDistanceMetric::kDistanceScale;
+
+    double score2 = distance_masked_images(source, source_y, source_x, target, target_y, target_x, m_patch_size);
+    double score = score1 * m_weight + score2 / (1 + m_weight);
+    return static_cast<int>(score / (1 + m_weight));
+}
+
+int RegularityGuidedPatchDistanceMetricV2::operator ()(const MaskedImage &source, int source_y, int source_x, const MaskedImage &target, int target_y, int target_x) const {
+    if (target_y < 0 || target_y >= target.size().height || target_x < 0 || target_x >= target.size().width)
+        return PatchDistanceMetric::kDistanceScale;
+
+    int source_scale = m_ijmap.size().height / source.size().height;
+    int target_scale = m_ijmap.size().height / target.size().height;
+
+    // fprintf(stderr, "RegularityGuidedPatchDistanceMetricV2 %d %d %d %d\n", source_y * source_scale, m_ijmap.size().height, source_x * source_scale, m_ijmap.size().width);
+
+    double score1 = PatchDistanceMetric::kDistanceScale;
+    if (!source.is_globally_masked(source_y, source_x) && !target.is_globally_masked(target_y, target_x)) {
+        auto source_ij = m_ijmap.ptr<float>(source_y * source_scale, source_x * source_scale);
+        auto target_ij = m_ijmap.ptr<float>(target_y * target_scale, target_x * target_scale);
+
+        float di = fabs(source_ij[0] - target_ij[0]); if (di > 0.5) di = 1 - di;
+        float dj = fabs(source_ij[1] - target_ij[1]); if (dj > 0.5) dj = 1 - dj;
+        score1 = sqrt(di * di + dj *dj) / 0.707;
+        if (score1 < 0 || score1 > 1) score1 = 1;
+        score1 *= PatchDistanceMetric::kDistanceScale;
+    }
+
+    double score2 = distance_masked_images(source, source_y, source_x, target, target_y, target_x, m_patch_size);
+    double score = score1 * m_weight + score2;
+    return int(score / (1 + m_weight));
+}
+

PyPatchMatch/csrc/nnf.h

@@ -0,0 +1,133 @@
+#pragma once
+
+#include <opencv2/core.hpp>
+#include "masked_image.h"
+
+class PatchDistanceMetric {
+public:
+    PatchDistanceMetric(int patch_size) : m_patch_size(patch_size) {}
+    virtual ~PatchDistanceMetric() = default;
+
+    inline int patch_size() const { return m_patch_size; }
+    virtual int operator()(const MaskedImage &source, int source_y, int source_x, const MaskedImage &target, int target_y, int target_x) const = 0;
+    static const int kDistanceScale;
+
+protected:
+    int m_patch_size;
+};
+
+class NearestNeighborField {
+public:
+    NearestNeighborField() : m_source(), m_target(), m_field(), m_distance_metric(nullptr) {
+        // pass
+    }
+    NearestNeighborField(const MaskedImage &source, const MaskedImage &target, const PatchDistanceMetric *metric, int max_retry = 20)
+        : m_source(source), m_target(target), m_distance_metric(metric) {
+        m_field = cv::Mat(m_source.size(), CV_32SC3);
+        _randomize_field(max_retry);
+    }
+    NearestNeighborField(const MaskedImage &source, const MaskedImage &target, const PatchDistanceMetric *metric, const NearestNeighborField &other, int max_retry = 20)
+            : m_source(source), m_target(target), m_distance_metric(metric) {
+        m_field = cv::Mat(m_source.size(), CV_32SC3);
+        _initialize_field_from(other, max_retry);
+    }
+
+    const MaskedImage &source() const {
+        return m_source;
+    }
+    const MaskedImage &target() const {
+        return m_target;
+    }
+    inline cv::Size source_size() const {
+        return m_source.size();
+    }
+    inline cv::Size target_size() const {
+        return m_target.size();
+    }
+    inline void set_source(const MaskedImage &source) {
+        m_source = source;
+    }
+    inline void set_target(const MaskedImage &target) {
+        m_target = target;
+    }
+
+    inline int *mutable_ptr(int y, int x) {
+        return m_field.ptr<int>(y, x);
+    }
+    inline const int *ptr(int y, int x) const {
+        return m_field.ptr<int>(y, x);
+    }
+
+    inline int at(int y, int x, int c) const {
+        return m_field.ptr<int>(y, x)[c];
+    }
+    inline int &at(int y, int x, int c) {
+        return m_field.ptr<int>(y, x)[c];
+    }
+    inline void set_identity(int y, int x) {
+        auto ptr = mutable_ptr(y, x);
+        ptr[0] = y, ptr[1] = x, ptr[2] = 0;
+    }
+
+    void minimize(int nr_pass);
+
+private:
+    inline int _distance(int source_y, int source_x, int target_y, int target_x) {
+        return (*m_distance_metric)(m_source, source_y, source_x, m_target, target_y, target_x);
+    }
+
+    void _randomize_field(int max_retry = 20, bool reset = true);
+    void _initialize_field_from(const NearestNeighborField &other, int max_retry);
+    void _minimize_link(int y, int x, int direction);
+
+    MaskedImage m_source;
+    MaskedImage m_target;
+    cv::Mat m_field;  // { y_target, x_target, distance_scaled }
+    const PatchDistanceMetric *m_distance_metric;
+};
+
+
+class PatchSSDDistanceMetric : public PatchDistanceMetric {
+public:
+    using PatchDistanceMetric::PatchDistanceMetric;
+    virtual int operator ()(const MaskedImage &source, int source_y, int source_x, const MaskedImage &target, int target_y, int target_x) const;
+    static const int kSSDScale;
+};
+
+class DebugPatchSSDDistanceMetric : public PatchDistanceMetric {
+public:
+    DebugPatchSSDDistanceMetric(int patch_size, int width, int height) : PatchDistanceMetric(patch_size), m_width(width), m_height(height) {}
+    virtual int operator ()(const MaskedImage &source, int source_y, int source_x, const MaskedImage &target, int target_y, int target_x) const;
+protected:
+    int m_width, m_height;
+};
+
+class RegularityGuidedPatchDistanceMetricV1 : public PatchDistanceMetric {
+public:
+    RegularityGuidedPatchDistanceMetricV1(int patch_size, double dx1, double dy1, double dx2, double dy2, double weight)
+        : PatchDistanceMetric(patch_size), m_dx1(dx1), m_dy1(dy1), m_dx2(dx2), m_dy2(dy2), m_weight(weight) {
+
+        assert(m_dy1 == 0);
+        assert(m_dx2 == 0);
+        m_scale = sqrt(m_dx1 * m_dx1 + m_dy2 * m_dy2) / 4;
+    }
+    virtual int operator ()(const MaskedImage &source, int source_y, int source_x, const MaskedImage &target, int target_y, int target_x) const;
+
+protected:
+    double m_dx1, m_dy1, m_dx2, m_dy2;
+    double m_scale, m_weight;
+};
+
+class RegularityGuidedPatchDistanceMetricV2 : public PatchDistanceMetric {
+public:
+    RegularityGuidedPatchDistanceMetricV2(int patch_size, cv::Mat ijmap, double weight)
+        : PatchDistanceMetric(patch_size), m_ijmap(ijmap), m_weight(weight) {
+
+    }
+    virtual int operator ()(const MaskedImage &source, int source_y, int source_x, const MaskedImage &target, int target_y, int target_x) const;
+
+protected:
+    cv::Mat m_ijmap;
+    double m_width, m_height, m_weight;
+};
+

PyPatchMatch/csrc/pyinterface.cpp

@@ -0,0 +1,107 @@
+#include "pyinterface.h"
+#include "inpaint.h"
+
+static unsigned int PM_seed = 1212;
+static bool PM_verbose = false;
+
+int _dtype_py_to_cv(int dtype_py);
+int _dtype_cv_to_py(int dtype_cv);
+cv::Mat _py_to_cv2(PM_mat_t pymat);
+PM_mat_t _cv2_to_py(cv::Mat cvmat);
+
+void PM_set_random_seed(unsigned int seed) {
+    PM_seed = seed;
+}
+
+void PM_set_verbose(int value) {
+    PM_verbose = static_cast<bool>(value);
+}
+
+void PM_free_pymat(PM_mat_t pymat) {
+    free(pymat.data_ptr);
+}
+
+PM_mat_t PM_inpaint(PM_mat_t source_py, PM_mat_t mask_py, int patch_size) {
+    cv::Mat source = _py_to_cv2(source_py);
+    cv::Mat mask = _py_to_cv2(mask_py);
+    auto metric = PatchSSDDistanceMetric(patch_size);
+    cv::Mat result = Inpainting(source, mask, &metric).run(PM_verbose, false, PM_seed);
+    return _cv2_to_py(result);
+}
+
+PM_mat_t PM_inpaint_regularity(PM_mat_t source_py, PM_mat_t mask_py, PM_mat_t ijmap_py, int patch_size, float guide_weight) {
+    cv::Mat source = _py_to_cv2(source_py);
+    cv::Mat mask = _py_to_cv2(mask_py);
+    cv::Mat ijmap = _py_to_cv2(ijmap_py);
+
+    auto metric = RegularityGuidedPatchDistanceMetricV2(patch_size, ijmap, guide_weight);
+    cv::Mat result = Inpainting(source, mask, &metric).run(PM_verbose, false, PM_seed);
+    return _cv2_to_py(result);
+}
+
+PM_mat_t PM_inpaint2(PM_mat_t source_py, PM_mat_t mask_py, PM_mat_t global_mask_py, int patch_size) {
+    cv::Mat source = _py_to_cv2(source_py);
+    cv::Mat mask = _py_to_cv2(mask_py);
+    cv::Mat global_mask = _py_to_cv2(global_mask_py);
+
+    auto metric = PatchSSDDistanceMetric(patch_size);
+    cv::Mat result = Inpainting(source, mask, global_mask, &metric).run(PM_verbose, false, PM_seed);
+    return _cv2_to_py(result);
+}
+
+PM_mat_t PM_inpaint2_regularity(PM_mat_t source_py, PM_mat_t mask_py, PM_mat_t global_mask_py, PM_mat_t ijmap_py, int patch_size, float guide_weight) {
+    cv::Mat source = _py_to_cv2(source_py);
+    cv::Mat mask = _py_to_cv2(mask_py);
+    cv::Mat global_mask = _py_to_cv2(global_mask_py);
+    cv::Mat ijmap = _py_to_cv2(ijmap_py);
+
+    auto metric = RegularityGuidedPatchDistanceMetricV2(patch_size, ijmap, guide_weight);
+    cv::Mat result = Inpainting(source, mask, global_mask, &metric).run(PM_verbose, false, PM_seed);
+    return _cv2_to_py(result);
+}
+
+int _dtype_py_to_cv(int dtype_py) {
+    switch (dtype_py) {
+        case PM_UINT8: return CV_8U;
+        case PM_INT8: return CV_8S;
+        case PM_UINT16: return CV_16U;
+        case PM_INT16: return CV_16S;
+        case PM_INT32: return CV_32S;
+        case PM_FLOAT32: return CV_32F;
+        case PM_FLOAT64: return CV_64F;
+    }
+
+    return CV_8U;
+}
+
+int _dtype_cv_to_py(int dtype_cv) {
+    switch (dtype_cv) {
+        case CV_8U: return PM_UINT8;
+        case CV_8S: return PM_INT8;
+        case CV_16U: return PM_UINT16;
+        case CV_16S: return PM_INT16;
+        case CV_32S: return PM_INT32;
+        case CV_32F: return PM_FLOAT32;
+        case CV_64F: return PM_FLOAT64;
+    }
+
+    return PM_UINT8;
+}
+
+cv::Mat _py_to_cv2(PM_mat_t pymat) {
+    int dtype = _dtype_py_to_cv(pymat.dtype);
+    dtype = CV_MAKETYPE(pymat.dtype, pymat.shape.channels);
+    return cv::Mat(cv::Size(pymat.shape.width, pymat.shape.height), dtype, pymat.data_ptr).clone();
+}
+
+PM_mat_t _cv2_to_py(cv::Mat cvmat) {
+    PM_shape_t shape = {cvmat.size().width, cvmat.size().height, cvmat.channels()};
+    int dtype = _dtype_cv_to_py(cvmat.depth());
+    size_t dsize = cvmat.total() * cvmat.elemSize();
+
+    void *data_ptr = reinterpret_cast<void *>(malloc(dsize));
+    memcpy(data_ptr, reinterpret_cast<void *>(cvmat.data), dsize);
+
+    return PM_mat_t {data_ptr, shape, dtype};
+}
+

PyPatchMatch/csrc/pyinterface.h

@@ -0,0 +1,38 @@
+#include <opencv2/core.hpp>
+#include <cstdlib>
+#include <cstdio>
+#include <cstring>
+
+extern "C" {
+
+struct PM_shape_t {
+    int width, height, channels;
+};
+
+enum PM_dtype_e {
+    PM_UINT8,
+    PM_INT8,
+    PM_UINT16,
+    PM_INT16,
+    PM_INT32,
+    PM_FLOAT32,
+    PM_FLOAT64,
+};
+
+struct PM_mat_t {
+    void *data_ptr;
+    PM_shape_t shape;
+    int dtype;
+};
+
+void PM_set_random_seed(unsigned int seed);
+void PM_set_verbose(int value);
+
+void PM_free_pymat(PM_mat_t pymat);
+PM_mat_t PM_inpaint(PM_mat_t image, PM_mat_t mask, int patch_size);
+PM_mat_t PM_inpaint_regularity(PM_mat_t image, PM_mat_t mask, PM_mat_t ijmap, int patch_size, float guide_weight);
+PM_mat_t PM_inpaint2(PM_mat_t image, PM_mat_t mask, PM_mat_t global_mask, int patch_size);
+PM_mat_t PM_inpaint2_regularity(PM_mat_t image, PM_mat_t mask, PM_mat_t global_mask, PM_mat_t ijmap, int patch_size, float guide_weight);
+
+} /*  extern "C" */
+

PyPatchMatch/examples/.gitignore

@@ -0,0 +1,2 @@
+/cpp_example.exe
+/images/*recovered.bmp

PyPatchMatch/examples/cpp_example.cpp

@@ -0,0 +1,31 @@
+#include <iostream>
+#include <opencv2/imgcodecs.hpp>
+#include <opencv2/highgui.hpp>
+
+#include "masked_image.h"
+#include "nnf.h"
+#include "inpaint.h"
+
+int main() {
+    auto source = cv::imread("./images/forest_pruned.bmp", cv::IMREAD_COLOR);
+
+    auto mask = cv::Mat(source.size(), CV_8UC1);
+    mask = cv::Scalar::all(0);
+    for (int i = 0; i < source.size().height; ++i) {
+        for (int j = 0; j < source.size().width; ++j) {
+            auto source_ptr = source.ptr<unsigned char>(i, j);
+            if (source_ptr[0] == 255 && source_ptr[1] == 255 && source_ptr[2] == 255) {
+                mask.at<unsigned char>(i, j) = 1;
+            }
+        }
+    }
+
+    auto metric = PatchSSDDistanceMetric(3);
+    auto result = Inpainting(source, mask, &metric).run(true, true);
+    // cv::imwrite("./images/forest_recovered.bmp", result);
+    // cv::imshow("Result", result);
+    // cv::waitKey();
+
+    return 0;
+}
+

PyPatchMatch/examples/cpp_example_run.sh

@@ -0,0 +1,18 @@
+#! /bin/bash
+#
+# cpp_example_run.sh
+# Copyright (C) 2020 Jiayuan Mao <maojiayuan@gmail.com>
+#
+# Distributed under terms of the MIT license.
+#
+
+set -x
+
+CFLAGS="-std=c++14 -O2 $(pkg-config --cflags opencv)"
+LDFLAGS="$(pkg-config --libs opencv)"
+g++ $CFLAGS cpp_example.cpp -I../csrc/ -L../ -lpatchmatch $LDFLAGS -o cpp_example.exe
+
+export DYLD_LIBRARY_PATH=../:$DYLD_LIBRARY_PATH  # For macOS
+export LD_LIBRARY_PATH=../:$LD_LIBRARY_PATH  # For Linux
+time ./cpp_example.exe
+

PyPatchMatch/examples/py_example.py

@@ -0,0 +1,21 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+# File   : test.py
+# Author : Jiayuan Mao
+# Email  : maojiayuan@gmail.com
+# Date   : 01/09/2020
+#
+# Distributed under terms of the MIT license.
+
+from PIL import Image
+
+import sys
+sys.path.insert(0, '../')
+import patch_match
+
+
+if __name__ == '__main__':
+    source = Image.open('./images/forest_pruned.bmp')
+    result = patch_match.inpaint(source, patch_size=3)
+    Image.fromarray(result).save('./images/forest_recovered.bmp')
+

PyPatchMatch/examples/py_example_global_mask.py

@@ -0,0 +1,27 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+# File   : test.py
+# Author : Jiayuan Mao
+# Email  : maojiayuan@gmail.com
+# Date   : 01/09/2020
+#
+# Distributed under terms of the MIT license.
+
+import numpy as np
+from PIL import Image
+
+import sys
+sys.path.insert(0, '../')
+import patch_match
+
+
+if __name__ == '__main__':
+    patch_match.set_verbose(True)
+    source = Image.open('./images/forest_pruned.bmp')
+    source = np.array(source)
+    source[:100, :100] = 255
+    global_mask = np.zeros_like(source[..., 0])
+    global_mask[:100, :100] = 1
+    result = patch_match.inpaint(source, global_mask=global_mask, patch_size=3)
+    Image.fromarray(result).save('./images/forest_recovered.bmp')
+

PyPatchMatch/patch_match.py

@@ -0,0 +1,201 @@
+#! /usr/bin/env python3
+# -*- coding: utf-8 -*-
+# File   : patch_match.py
+# Author : Jiayuan Mao
+# Email  : maojiayuan@gmail.com
+# Date   : 01/09/2020
+#
+# Distributed under terms of the MIT license.
+
+import ctypes
+import os.path as osp
+from typing import Optional, Union
+
+import numpy as np
+from PIL import Image
+
+try:
+    # If the Jacinle library (https://github.com/vacancy/Jacinle) is present, use its auto_travis feature.
+    from jacinle.jit.cext import auto_travis
+    auto_travis(__file__, required_files=['*.so'])
+except ImportError as e:
+    # Otherwise, fall back to the subprocess.
+    import subprocess
+    print('Compiling and loading c extensions from "{}".'.format(osp.realpath(osp.dirname(__file__))))
+    subprocess.check_call(['./travis.sh'], cwd=osp.dirname(__file__))
+
+
+__all__ = ['set_random_seed', 'set_verbose', 'inpaint', 'inpaint_regularity']
+
+
+class CShapeT(ctypes.Structure):
+    _fields_ = [
+        ('width', ctypes.c_int),
+        ('height', ctypes.c_int),
+        ('channels', ctypes.c_int),
+    ]
+
+
+class CMatT(ctypes.Structure):
+    _fields_ = [
+        ('data_ptr', ctypes.c_void_p),
+        ('shape', CShapeT),
+        ('dtype', ctypes.c_int)
+    ]
+
+
+PMLIB = ctypes.CDLL(osp.join(osp.dirname(__file__), 'libpatchmatch.so'))
+
+PMLIB.PM_set_random_seed.argtypes = [ctypes.c_uint]
+PMLIB.PM_set_verbose.argtypes = [ctypes.c_int]
+PMLIB.PM_free_pymat.argtypes = [CMatT]
+PMLIB.PM_inpaint.argtypes = [CMatT, CMatT, ctypes.c_int]
+PMLIB.PM_inpaint.restype = CMatT
+PMLIB.PM_inpaint_regularity.argtypes = [CMatT, CMatT, CMatT, ctypes.c_int, ctypes.c_float]
+PMLIB.PM_inpaint_regularity.restype = CMatT
+PMLIB.PM_inpaint2.argtypes = [CMatT, CMatT, CMatT, ctypes.c_int]
+PMLIB.PM_inpaint2.restype = CMatT
+PMLIB.PM_inpaint2_regularity.argtypes = [CMatT, CMatT, CMatT, CMatT, ctypes.c_int, ctypes.c_float]
+PMLIB.PM_inpaint2_regularity.restype = CMatT
+
+
+def set_random_seed(seed: int):
+    PMLIB.PM_set_random_seed(ctypes.c_uint(seed))
+
+
+def set_verbose(verbose: bool):
+    PMLIB.PM_set_verbose(ctypes.c_int(verbose))
+
+
+def inpaint(
+    image: Union[np.ndarray, Image.Image],
+    mask: Optional[Union[np.ndarray, Image.Image]] = None,
+    *,
+    global_mask: Optional[Union[np.ndarray, Image.Image]] = None,
+    patch_size: int = 15
+) -> np.ndarray:
+    """
+    PatchMatch based inpainting proposed in:
+
+        PatchMatch : A Randomized Correspondence Algorithm for Structural Image Editing
+        C.Barnes, E.Shechtman, A.Finkelstein and Dan B.Goldman
+        SIGGRAPH 2009
+
+    Args:
+        image (Union[np.ndarray, Image.Image]): the input image, should be 3-channel RGB/BGR.
+        mask (Union[np.array, Image.Image], optional): the mask of the hole(s) to be filled, should be 1-channel.
+        If not provided (None), the algorithm will treat all purely white pixels as the holes (255, 255, 255).
+        global_mask (Union[np.array, Image.Image], optional): the target mask of the output image.
+        patch_size (int): the patch size for the inpainting algorithm.
+
+    Return:
+        result (np.ndarray): the repaired image, of the same size as the input image.
+    """
+
+    if isinstance(image, Image.Image):
+        image = np.array(image)
+    image = np.ascontiguousarray(image)
+    assert image.ndim == 3 and image.shape[2] == 3 and image.dtype == 'uint8'
+
+    if mask is None:
+        mask = (image == (255, 255, 255)).all(axis=2, keepdims=True).astype('uint8')
+        mask = np.ascontiguousarray(mask)
+    else:
+        mask = _canonize_mask_array(mask)
+
+    if global_mask is None:
+        ret_pymat = PMLIB.PM_inpaint(np_to_pymat(image), np_to_pymat(mask), ctypes.c_int(patch_size))
+    else:
+        global_mask = _canonize_mask_array(global_mask)
+        ret_pymat = PMLIB.PM_inpaint2(np_to_pymat(image), np_to_pymat(mask), np_to_pymat(global_mask), ctypes.c_int(patch_size))
+
+    ret_npmat = pymat_to_np(ret_pymat)
+    PMLIB.PM_free_pymat(ret_pymat)
+
+    return ret_npmat
+
+
+def inpaint_regularity(
+    image: Union[np.ndarray, Image.Image],
+    mask: Optional[Union[np.ndarray, Image.Image]],
+    ijmap: np.ndarray,
+    *,
+    global_mask: Optional[Union[np.ndarray, Image.Image]] = None,
+    patch_size: int = 15, guide_weight: float = 0.25
+) -> np.ndarray:
+    if isinstance(image, Image.Image):
+        image = np.array(image)
+    image = np.ascontiguousarray(image)
+
+    assert isinstance(ijmap, np.ndarray) and ijmap.ndim == 3 and ijmap.shape[2] == 3 and ijmap.dtype == 'float32'
+    ijmap = np.ascontiguousarray(ijmap)
+
+    assert image.ndim == 3 and image.shape[2] == 3 and image.dtype == 'uint8'
+    if mask is None:
+        mask = (image == (255, 255, 255)).all(axis=2, keepdims=True).astype('uint8')
+        mask = np.ascontiguousarray(mask)
+    else:
+        mask = _canonize_mask_array(mask)
+
+
+    if global_mask is None:
+        ret_pymat = PMLIB.PM_inpaint_regularity(np_to_pymat(image), np_to_pymat(mask), np_to_pymat(ijmap), ctypes.c_int(patch_size), ctypes.c_float(guide_weight))
+    else:
+        global_mask = _canonize_mask_array(global_mask)
+        ret_pymat = PMLIB.PM_inpaint2_regularity(np_to_pymat(image), np_to_pymat(mask), np_to_pymat(global_mask), np_to_pymat(ijmap), ctypes.c_int(patch_size), ctypes.c_float(guide_weight))
+
+    ret_npmat = pymat_to_np(ret_pymat)
+    PMLIB.PM_free_pymat(ret_pymat)
+
+    return ret_npmat
+
+
+def _canonize_mask_array(mask):
+    if isinstance(mask, Image.Image):
+        mask = np.array(mask)
+    if mask.ndim == 2 and mask.dtype == 'uint8':
+        mask = mask[..., np.newaxis]
+    assert mask.ndim == 3 and mask.shape[2] == 1 and mask.dtype == 'uint8'
+    return np.ascontiguousarray(mask)
+
+
+dtype_pymat_to_ctypes = [
+    ctypes.c_uint8,
+    ctypes.c_int8,
+    ctypes.c_uint16,
+    ctypes.c_int16,
+    ctypes.c_int32,
+    ctypes.c_float,
+    ctypes.c_double,
+]
+
+
+dtype_np_to_pymat = {
+    'uint8': 0,
+    'int8': 1,
+    'uint16': 2,
+    'int16': 3,
+    'int32': 4,
+    'float32': 5,
+    'float64': 6,
+}
+
+
+def np_to_pymat(npmat):
+    assert npmat.ndim == 3
+    return CMatT(
+        ctypes.cast(npmat.ctypes.data, ctypes.c_void_p),
+        CShapeT(npmat.shape[1], npmat.shape[0], npmat.shape[2]),
+        dtype_np_to_pymat[str(npmat.dtype)]
+    )
+
+
+def pymat_to_np(pymat):
+    npmat = np.ctypeslib.as_array(
+        ctypes.cast(pymat.data_ptr, ctypes.POINTER(dtype_pymat_to_ctypes[pymat.dtype])),
+        (pymat.shape.height, pymat.shape.width, pymat.shape.channels)
+    )
+    ret = np.empty(npmat.shape, npmat.dtype)
+    ret[:] = npmat
+    return ret
+

PyPatchMatch/travis.sh

@@ -0,0 +1,9 @@
+#! /bin/bash
+#
+# travis.sh
+# Copyright (C) 2020 Jiayuan Mao <maojiayuan@gmail.com>
+#
+# Distributed under terms of the MIT license.
+#
+
+make clean && make

app.py

@@ -0,0 +1,390 @@
+import io
+import base64
+import os
+
+import numpy as np
+import torch
+from torch import autocast
+from diffusers import StableDiffusionPipeline, StableDiffusionInpaintPipeline
+from PIL import Image
+from PIL import ImageOps
+import gradio as gr
+import base64
+import skimage
+import skimage.measure
+from utils import *
+
+
+def load_html():
+    body, canvaspy = "", ""
+    with open("index.html", encoding="utf8") as f:
+        body = f.read()
+    with open("canvas.py", encoding="utf8") as f:
+        canvaspy = f.read()
+    body = body.replace("- paths:\n", "")
+    body = body.replace("  - ./canvas.py\n", "")
+    body = body.replace("from canvas import InfCanvas", canvaspy)
+    return body
+
+
+def test(x):
+    x = load_html()
+    return f"""<iframe id="sdinfframe" style="width: 100%; height: 700px" name="result" allow="midi; geolocation; microphone; camera; 
+    display-capture; encrypted-media;" sandbox="allow-modals allow-forms 
+    allow-scripts allow-same-origin allow-popups 
+    allow-top-navigation-by-user-activation allow-downloads" allowfullscreen="" 
+    allowpaymentrequest="" frameborder="0" srcdoc='{x}'></iframe>"""
+
+
+DEBUG_MODE = False
+
+try:
+    SAMPLING_MODE = Image.Resampling.LANCZOS
+except Exception as e:
+    SAMPLING_MODE = Image.LANCZOS
+
+try:
+    contain_func = ImageOps.contain
+except Exception as e:
+
+    def contain_func(image, size, method=SAMPLING_MODE):
+        # from PIL: https://pillow.readthedocs.io/en/stable/reference/ImageOps.html#PIL.ImageOps.contain
+        im_ratio = image.width / image.height
+        dest_ratio = size[0] / size[1]
+        if im_ratio != dest_ratio:
+            if im_ratio > dest_ratio:
+                new_height = int(image.height / image.width * size[0])
+                if new_height != size[1]:
+                    size = (size[0], new_height)
+            else:
+                new_width = int(image.width / image.height * size[1])
+                if new_width != size[0]:
+                    size = (new_width, size[1])
+        return image.resize(size, resample=method)
+
+
+PAINT_SELECTION = "✥"
+IMAGE_SELECTION = "🖼️"
+BRUSH_SELECTION = "🖌️"
+blocks = gr.Blocks()
+model = {}
+model["width"] = 1500
+model["height"] = 600
+model["sel_size"] = 256
+
+def get_token():
+    token = ""
+    token = os.environ.get("hftoken", token)
+    return token
+
+
+def save_token(token):
+    return
+
+
+def get_model(token=""):
+    if "text2img" not in model:
+        text2img = StableDiffusionPipeline.from_pretrained(
+            "CompVis/stable-diffusion-v1-4",
+            revision="fp16",
+            torch_dtype=torch.float16,
+            use_auth_token=token,
+        ).to("cuda")
+        model["safety_checker"] = text2img.safety_checker
+        inpaint = StableDiffusionInpaintPipeline(
+            vae=text2img.vae,
+            text_encoder=text2img.text_encoder,
+            tokenizer=text2img.tokenizer,
+            unet=text2img.unet,
+            scheduler=text2img.scheduler,
+            safety_checker=text2img.safety_checker,
+            feature_extractor=text2img.feature_extractor,
+        ).to("cuda")
+        save_token(token)
+        try:
+            total_memory = torch.cuda.get_device_properties(0).total_memory // (
+                1024 ** 3
+            )
+            if total_memory <= 5:
+                inpaint.enable_attention_slicing()
+        except:
+            pass
+        model["text2img"] = text2img
+        model["inpaint"] = inpaint
+    return model["text2img"], model["inpaint"]
+
+
+def run_outpaint(
+    sel_buffer_str,
+    prompt_text,
+    strength,
+    guidance,
+    step,
+    resize_check,
+    fill_mode,
+    enable_safety,
+    state,
+):
+    base64_str = "base64"
+    if True:
+        text2img, inpaint = get_model()
+        if enable_safety:
+            text2img.safety_checker = model["safety_checker"]
+            inpaint.safety_checker = model["safety_checker"]
+        else:
+            text2img.safety_checker = lambda images, **kwargs: (images, False)
+            inpaint.safety_checker = lambda images, **kwargs: (images, False)
+        data = base64.b64decode(str(sel_buffer_str))
+        pil = Image.open(io.BytesIO(data))
+        # base.output.clear_output()
+        # base.read_selection_from_buffer()
+        sel_buffer = np.array(pil)
+        img = sel_buffer[:, :, 0:3]
+        mask = sel_buffer[:, :, -1]
+        process_size = 512 if resize_check else model["sel_size"]
+        if mask.sum() > 0:
+            img, mask = functbl[fill_mode](img, mask)
+            init_image = Image.fromarray(img)
+            mask = 255 - mask
+            mask = skimage.measure.block_reduce(mask, (8, 8), np.max)
+            mask = mask.repeat(8, axis=0).repeat(8, axis=1)
+            mask_image = Image.fromarray(mask)
+            # mask_image=mask_image.filter(ImageFilter.GaussianBlur(radius = 8))
+            with autocast("cuda"):
+                images = inpaint(
+                    prompt=prompt_text,
+                    init_image=init_image.resize(
+                        (process_size, process_size), resample=SAMPLING_MODE
+                    ),
+                    mask_image=mask_image.resize((process_size, process_size)),
+                    strength=strength,
+                    num_inference_steps=step,
+                    guidance_scale=guidance,
+                )["sample"]
+        else:
+            with autocast("cuda"):
+                images = text2img(
+                    prompt=prompt_text, height=process_size, width=process_size,
+                )["sample"]
+        out = sel_buffer.copy()
+        out[:, :, 0:3] = np.array(
+            images[0].resize(
+                (model["sel_size"], model["sel_size"]), resample=SAMPLING_MODE,
+            )
+        )
+        out[:, :, -1] = 255
+        out_pil = Image.fromarray(out)
+        out_buffer = io.BytesIO()
+        out_pil.save(out_buffer, format="PNG")
+        out_buffer.seek(0)
+        base64_bytes = base64.b64encode(out_buffer.read())
+        base64_str = base64_bytes.decode("ascii")
+    return (
+        gr.update(label=str(state + 1), value=base64_str,),
+        gr.update(label="Prompt"),
+        state + 1,
+    )
+
+
+def load_js(name):
+    if name in ["export", "commit", "undo"]:
+        return f"""
+function (x)
+{{ 
+    let frame=document.querySelector("gradio-app").shadowRoot.querySelector("#sdinfframe").contentWindow.document;
+    let button=frame.querySelector("#{name}");
+    button.click();
+    return x;
+}}
+"""
+    ret = ""
+    with open(f"./js/{name}.js", "r") as f:
+        ret = f.read()
+    return ret
+
+
+upload_button_js = load_js("upload")
+outpaint_button_js = load_js("outpaint")
+proceed_button_js = load_js("proceed")
+mode_js = load_js("mode")
+setup_button_js = load_js("setup")
+
+get_model(get_token())
+
+with blocks as demo:
+    # title
+    title = gr.Markdown(
+        """
+    **stablediffusion-infinity**: Outpainting with Stable Diffusion on an infinite canvas: [https://github.com/lkwq007/stablediffusion-infinity](https://github.com/lkwq007/stablediffusion-infinity)
+    """
+    )
+    # frame
+    frame = gr.HTML(test(2), visible=True)
+    # setup
+    # with gr.Row():
+    #     token = gr.Textbox(
+    #         label="Huggingface token",
+    #         value="",
+    #         placeholder="Input your token here",
+    #     )
+    #     canvas_width = gr.Number(
+    #         label="Canvas width", value=1024, precision=0, elem_id="canvas_width"
+    #     )
+    #     canvas_height = gr.Number(
+    #         label="Canvas height", value=600, precision=0, elem_id="canvas_height"
+    #     )
+    #     selection_size = gr.Number(
+    #         label="Selection box size", value=256, precision=0, elem_id="selection_size"
+    #     )
+    # setup_button = gr.Button("Start (may take a while)", variant="primary")
+    with gr.Row():
+        with gr.Column(scale=3, min_width=270):
+            # canvas control
+            canvas_control = gr.Radio(
+                label="Control",
+                choices=[PAINT_SELECTION, IMAGE_SELECTION, BRUSH_SELECTION],
+                value=PAINT_SELECTION,
+                elem_id="control",
+            )
+            with gr.Box():
+                with gr.Group():
+                    run_button = gr.Button(value="Outpaint")
+                    export_button = gr.Button(value="Export")
+                    commit_button = gr.Button(value="✓")
+                    retry_button = gr.Button(value="⟳")
+                    undo_button = gr.Button(value="↶")
+        with gr.Column(scale=3, min_width=270):
+            sd_prompt = gr.Textbox(
+                label="Prompt", placeholder="input your prompt here", lines=4
+            )
+        with gr.Column(scale=2, min_width=150):
+            with gr.Box():
+                sd_resize = gr.Checkbox(label="Resize input to 515x512", value=True)
+                safety_check = gr.Checkbox(label="Enable Safety Checker", value=True)
+            sd_strength = gr.Slider(
+                label="Strength", minimum=0.0, maximum=1.0, value=0.75, step=0.01
+            )
+        with gr.Column(scale=1, min_width=150):
+            sd_step = gr.Number(label="Step", value=50, precision=0)
+            sd_guidance = gr.Number(label="Guidance", value=7.5)
+    with gr.Row():
+        with gr.Column(scale=4, min_width=600):
+            init_mode = gr.Radio(
+                label="Init mode",
+                choices=[
+                    "patchmatch",
+                    "edge_pad",
+                    "cv2_ns",
+                    "cv2_telea",
+                    "gaussian",
+                    "perlin",
+                ],
+                value="patchmatch",
+                type="value",
+            )
+
+    proceed_button = gr.Button("Proceed", elem_id="proceed", visible=DEBUG_MODE)
+    # sd pipeline parameters
+    with gr.Accordion("Upload image", open=False):
+        image_box = gr.Image(image_mode="RGBA", source="upload", type="pil")
+        upload_button = gr.Button(
+            "Upload"
+        )
+    model_output = gr.Textbox(visible=DEBUG_MODE, elem_id="output", label="0")
+    model_input = gr.Textbox(visible=DEBUG_MODE, elem_id="input", label="Input")
+    upload_output = gr.Textbox(visible=DEBUG_MODE, elem_id="upload", label="0")
+    model_output_state = gr.State(value=0)
+    upload_output_state = gr.State(value=0)
+    # canvas_state = gr.State({"width":1024,"height":600,"selection_size":384})
+
+    def upload_func(image, state):
+        pil = image.convert("RGBA")
+        w, h = pil.size
+        if w > model["width"] - 100 or h > model["height"] - 100:
+            pil = contain_func(pil, (model["width"] - 100, model["height"] - 100))
+        out_buffer = io.BytesIO()
+        pil.save(out_buffer, format="PNG")
+        out_buffer.seek(0)
+        base64_bytes = base64.b64encode(out_buffer.read())
+        base64_str = base64_bytes.decode("ascii")
+        return (
+            gr.update(label=str(state + 1), value=base64_str),
+            state + 1,
+        )
+
+    upload_button.click(
+        fn=upload_func,
+        inputs=[image_box, upload_output_state],
+        outputs=[upload_output, upload_output_state],
+        _js=upload_button_js,
+    )
+
+    def setup_func(token_val, width, height, size):
+        model["width"] = width
+        model["height"] = height
+        model["sel_size"] = size
+        try:
+            get_model(token_val)
+        except Exception as e:
+            return {token: gr.update(value="Invalid token!")}
+        return {
+            token: gr.update(visible=False),
+            canvas_width: gr.update(visible=False),
+            canvas_height: gr.update(visible=False),
+            selection_size: gr.update(visible=False),
+            setup_button: gr.update(visible=False),
+            frame: gr.update(visible=True),
+            upload_button: gr.update(value="Upload"),
+        }
+
+    # setup_button.click(
+    #     fn=setup_func,
+    #     inputs=[token, canvas_width, canvas_height, selection_size],
+    #     outputs=[
+    #         token,
+    #         canvas_width,
+    #         canvas_height,
+    #         selection_size,
+    #         setup_button,
+    #         frame,
+    #         upload_button,
+    #     ],
+    #     _js=setup_button_js,
+    # )
+    run_button.click(
+        fn=None, inputs=[run_button], outputs=[run_button], _js=outpaint_button_js,
+    )
+    retry_button.click(
+        fn=None, inputs=[run_button], outputs=[run_button], _js=outpaint_button_js,
+    )
+    proceed_button.click(
+        fn=run_outpaint,
+        inputs=[
+            model_input,
+            sd_prompt,
+            sd_strength,
+            sd_guidance,
+            sd_step,
+            sd_resize,
+            init_mode,
+            safety_check,
+            model_output_state,
+        ],
+        outputs=[model_output, sd_prompt, model_output_state],
+        _js=proceed_button_js,
+    )
+    export_button.click(
+        fn=None, inputs=[export_button], outputs=[export_button], _js=load_js("export")
+    )
+    commit_button.click(
+        fn=None, inputs=[export_button], outputs=[export_button], _js=load_js("commit")
+    )
+    undo_button.click(
+        fn=None, inputs=[export_button], outputs=[export_button], _js=load_js("undo")
+    )
+    canvas_control.change(
+        fn=None, inputs=[canvas_control], outputs=[canvas_control], _js=mode_js,
+    )
+
+demo.launch()
+

canvas.py

@@ -0,0 +1,547 @@
+import base64
+import io
+import numpy as np
+from PIL import Image
+from pyodide import to_js, create_proxy
+from js import (
+    console,
+    document,
+    devicePixelRatio,
+    ImageData,
+    Uint8ClampedArray,
+    CanvasRenderingContext2D as Context2d,
+    requestAnimationFrame,
+)
+
+PAINT_SELECTION = "✥"
+IMAGE_SELECTION = "🖼️"
+BRUSH_SELECTION = "🖌️"
+NOP_MODE = 0
+PAINT_MODE = 1
+IMAGE_MODE = 2
+BRUSH_MODE = 3
+
+
+def hold_canvas():
+    pass
+
+
+def prepare_canvas(width, height, canvas) -> Context2d:
+    ctx = canvas.getContext("2d")
+
+    canvas.style.width = f"{width}px"
+    canvas.style.height = f"{height}px"
+
+    canvas.width = width
+    canvas.height = height
+
+    ctx.clearRect(0, 0, width, height)
+
+    return ctx
+
+
+# class MultiCanvas:
+#     def __init__(self,layer,width=800, height=600) -> None:
+#         pass
+def multi_canvas(layer, width=800, height=600):
+    lst = [
+        CanvasProxy(document.querySelector(f"#canvas{i}"), width, height)
+        for i in range(layer)
+    ]
+    return lst
+
+
+class CanvasProxy:
+    def __init__(self, canvas, width=800, height=600) -> None:
+        self.canvas = canvas
+        self.ctx = prepare_canvas(width, height, canvas)
+        self.width = width
+        self.height = height
+
+    def clear_rect(self, x, y, w, h):
+        self.ctx.clearRect(x, y, w, h)
+
+    def clear(self,):
+        self.clear_rect(0, 0, self.width, self.height)
+
+    def stroke_rect(self, x, y, w, h):
+        self.ctx.strokeRect(x, y, w, h)
+
+    def fill_rect(self, x, y, w, h):
+        self.ctx.fillRect(x, y, w, h)
+
+    def put_image_data(self, image, x, y):
+        data = Uint8ClampedArray.new(to_js(image.tobytes()))
+        height, width, _ = image.shape
+        image_data = ImageData.new(data, width, height)
+        self.ctx.putImageData(image_data, x, y)
+
+    @property
+    def stroke_style(self):
+        return self.ctx.strokeStyle
+
+    @stroke_style.setter
+    def stroke_style(self, value):
+        self.ctx.strokeStyle = value
+
+    @property
+    def fill_style(self):
+        return self.ctx.strokeStyle
+
+    @fill_style.setter
+    def fill_style(self, value):
+        self.ctx.fillStyle = value
+
+
+# RGBA for masking
+class InfCanvas:
+    def __init__(
+        self,
+        width,
+        height,
+        selection_size=256,
+        grid_size=32,
+        patch_size=4096,
+        test_mode=False,
+    ) -> None:
+        assert selection_size < min(height, width)
+        self.width = width
+        self.height = height
+        self.canvas = multi_canvas(5, width=width, height=height)
+        # self.canvas = Canvas(width=width, height=height)
+        self.view_pos = [0, 0]
+        self.cursor = [
+            width // 2 - selection_size // 2,
+            height // 2 - selection_size // 2,
+        ]
+        self.data = {}
+        self.grid_size = grid_size
+        self.selection_size = selection_size
+        self.patch_size = patch_size
+        # note that for image data, the height comes before width
+        self.buffer = np.zeros((height, width, 4), dtype=np.uint8)
+        self.sel_buffer = np.zeros((selection_size, selection_size, 4), dtype=np.uint8)
+        self.sel_buffer_bak = np.zeros(
+            (selection_size, selection_size, 4), dtype=np.uint8
+        )
+        self.sel_dirty = False
+        self.buffer_dirty = False
+        self.mouse_pos = [-1, -1]
+        self.mouse_state = 0
+        # self.output = widgets.Output()
+        self.test_mode = test_mode
+        self.buffer_updated = False
+        self.image_move_freq = 1
+        self.show_brush = False
+        # inpaint pipeline from diffuser
+
+    def setup_mouse(self):
+        self.image_move_cnt = 0
+
+        def get_mouse_mode():
+            mode = document.querySelector("#mode").value
+            if mode == PAINT_SELECTION:
+                return PAINT_MODE
+            elif mode == IMAGE_SELECTION:
+                return IMAGE_MODE
+            return BRUSH_MODE
+
+        def get_event_pos(event):
+            canvas = self.canvas[-1].canvas
+            rect = canvas.getBoundingClientRect()
+            x = (canvas.width * (event.clientX - rect.left)) / rect.width
+            y = (canvas.height * (event.clientY - rect.top)) / rect.height
+            return x, y
+
+        def handle_mouse_down(event):
+            self.mouse_state = get_mouse_mode()
+
+        def handle_mouse_out(event):
+            last_state = self.mouse_state
+            self.mouse_state = NOP_MODE
+            self.image_move_cnt = 0
+            if last_state == IMAGE_MODE:
+                if True:
+                    self.clear_background()
+                    self.draw_buffer()
+                    self.canvas[2].clear()
+                    self.draw_selection_box()
+            if self.show_brush:
+                self.canvas[-2].clear()
+                self.show_brush = False
+
+        def handle_mouse_up(event):
+            last_state = self.mouse_state
+            self.mouse_state = NOP_MODE
+            self.image_move_cnt = 0
+            if last_state == IMAGE_MODE:
+                if True:
+                    self.clear_background()
+                    self.draw_buffer()
+                    self.canvas[2].clear()
+                    self.draw_selection_box()
+
+        async def handle_mouse_move(event):
+            x, y = get_event_pos(event)
+            x0, y0 = self.mouse_pos
+            xo = x - x0
+            yo = y - y0
+            if self.mouse_state == PAINT_MODE:
+                self.update_cursor(int(xo), int(yo))
+                if True:
+                    # self.clear_background()
+                    # console.log(self.buffer_updated)
+                    if self.buffer_updated:
+                        self.draw_buffer()
+                        self.buffer_updated = False
+                    self.draw_selection_box()
+            elif self.mouse_state == IMAGE_MODE:
+                self.image_move_cnt += 1
+                self.update_view_pos(int(xo), int(yo))
+                if self.image_move_cnt == self.image_move_freq:
+                    if True:
+                        self.clear_background()
+                        self.draw_buffer()
+                        self.canvas[2].clear()
+                        self.draw_selection_box()
+                    self.image_move_cnt = 0
+            elif self.mouse_state == BRUSH_MODE:
+                if self.sel_dirty:
+                    self.write_selection_to_buffer()
+                    self.canvas[2].clear()
+                self.buffer_dirty=True
+                bx0,by0=int(x)-self.grid_size//2,int(y)-self.grid_size//2
+                bx1,by1=bx0+self.grid_size,by0+self.grid_size
+                bx0,by0=max(0,bx0),max(0,by0)
+                bx1,by1=min(self.width,bx1),min(self.height,by1)
+                self.buffer[by0:by1,bx0:bx1,:]*=0
+                self.draw_buffer()
+                self.draw_selection_box()
+
+            mode = document.querySelector("#mode").value
+            if mode == BRUSH_SELECTION:
+                self.canvas[-2].clear()
+                self.canvas[-2].fill_style = "#ffffff"
+                self.canvas[-2].fill_rect(x-self.grid_size//2,y-self.grid_size//2,self.grid_size,self.grid_size)
+                self.canvas[-2].stroke_rect(x-self.grid_size//2,y-self.grid_size//2,self.grid_size,self.grid_size)
+                self.show_brush = True
+            elif self.show_brush:
+                self.canvas[-2].clear()
+                self.show_brush = False
+            self.mouse_pos[0] = x
+            self.mouse_pos[1] = y
+
+        self.canvas[-1].canvas.addEventListener(
+            "mousedown", create_proxy(handle_mouse_down)
+        )
+        self.canvas[-1].canvas.addEventListener(
+            "mousemove", create_proxy(handle_mouse_move)
+        )
+        self.canvas[-1].canvas.addEventListener(
+            "mouseup", create_proxy(handle_mouse_up)
+        )
+        self.canvas[-1].canvas.addEventListener(
+            "mouseout", create_proxy(handle_mouse_out)
+        )
+
+    def setup_widgets(self):
+        self.mode_button = widgets.ToggleButtons(
+            options=[PAINT_SELECTION, IMAGE_SELECTION],
+            disabled=False,
+            button_style="",
+            style={"button_width": "50px", "font_weight": "bold"},
+            tooltips=["Outpaint region", "Image"],
+        )
+        self.test_button = widgets.ToggleButtons(
+            options=["r", "g", "b"],
+            disabled=False,
+            style={"button_width": "50px", "font_weight": "bold", "font_size": "36px"},
+        )
+        self.text_input = widgets.Textarea(
+            value="",
+            placeholder="input your prompt here",
+            description="Prompt:",
+            disabled=False,
+        )
+        self.run_button = widgets.Button(
+            description="Outpaint",
+            tooltip="Run outpainting",
+            icon="pen",
+            button_style="primary",
+        )
+        self.export_button = widgets.Button(
+            description="Export",
+            tooltip="Export the image",
+            icon="save",
+            button_style="success",
+        )
+        self.fill_button = widgets.ToggleButtons(
+            description="Init mode:",
+            options=[
+                "patchmatch",
+                "edge_pad",
+                "cv2_ns",
+                "cv2_telea",
+                "gaussian",
+                "perlin",
+            ],
+            disabled=False,
+            button_style="",
+            style={"button_width": "80px", "font_weight": "bold"},
+        )
+
+        if self.test_mode:
+
+            def test_button_clicked(btn):
+                # lst.append(tuple(base.cursor))
+                with self.output:
+                    val = self.test_button.value
+                    if val == "r":
+                        self.fill_selection(
+                            np.tile(
+                                np.array([255, 0, 0, 255], dtype=np.uint8),
+                                (self.selection_size, self.selection_size, 1),
+                            )
+                        )
+                    if val == "g":
+                        self.fill_selection(
+                            np.tile(
+                                np.array([0, 255, 0, 255], dtype=np.uint8),
+                                (self.selection_size, self.selection_size, 1),
+                            )
+                        )
+                    if val == "b":
+                        self.fill_selection(
+                            np.tile(
+                                np.array([0, 0, 255, 255], dtype=np.uint8),
+                                (self.selection_size, self.selection_size, 1),
+                            )
+                        )
+                    if True:
+                        self.clear_background()
+                        self.draw_buffer()
+                        self.draw_selection_box()
+
+            self.run_button.on_click(test_button_clicked)
+
+    def display(self):
+        if True:
+            self.clear_background()
+            self.draw_buffer()
+            self.draw_selection_box()
+        if self.test_mode:
+            return [
+                self.test_button,
+                self.mode_button,
+                self.canvas,
+                widgets.HBox([self.run_button, self.text_input]),
+                self.output,
+            ]
+        return [
+            self.fill_button,
+            self.canvas,
+            widgets.HBox(
+                [self.mode_button, self.run_button, self.export_button, self.text_input]
+            ),
+            self.output,
+        ]
+
+    def clear_background(self):
+        # fake transparent background
+        h, w, step = self.height, self.width, self.grid_size
+        stride = step * 2
+        x0, y0 = self.view_pos
+        x0 = (-x0) % stride
+        y0 = (-y0) % stride
+        # self.canvas.clear()
+        self.canvas[0].fill_style = "#ffffff"
+        self.canvas[0].fill_rect(0, 0, w, h)
+        self.canvas[0].fill_style = "#aaaaaa"
+        for y in range(y0 - stride, h + step, step):
+            start = (x0 - stride) if y // step % 2 == 0 else (x0 - step)
+            for x in range(start, w + step, stride):
+                self.canvas[0].fill_rect(x, y, step, step)
+        self.canvas[0].stroke_rect(0, 0, w, h)
+
+    def update_view_pos(self, xo, yo):
+        if abs(xo) + abs(yo) == 0:
+            return
+        if self.sel_dirty:
+            self.write_selection_to_buffer()
+        if self.buffer_dirty:
+            self.buffer2data()
+        self.view_pos[0] -= xo
+        self.view_pos[1] -= yo
+        self.data2buffer()
+        # self.read_selection_from_buffer()
+
+    def update_cursor(self, xo, yo):
+        if abs(xo) + abs(yo) == 0:
+            return
+        if self.sel_dirty:
+            self.write_selection_to_buffer()
+        self.cursor[0] += xo
+        self.cursor[1] += yo
+        self.cursor[0] = max(min(self.width - self.selection_size, self.cursor[0]), 0)
+        self.cursor[1] = max(min(self.height - self.selection_size, self.cursor[1]), 0)
+        # self.read_selection_from_buffer()
+
+    def data2buffer(self):
+        x, y = self.view_pos
+        h, w = self.height, self.width
+        # fill four parts
+        for i in range(4):
+            pos_src, pos_dst, data = self.select(x, y, i)
+            xs0, xs1 = pos_src[0]
+            ys0, ys1 = pos_src[1]
+            xd0, xd1 = pos_dst[0]
+            yd0, yd1 = pos_dst[1]
+            self.buffer[yd0:yd1, xd0:xd1, :] = data[ys0:ys1, xs0:xs1, :]
+
+    def buffer2data(self):
+        x, y = self.view_pos
+        h, w = self.height, self.width
+        # fill four parts
+        for i in range(4):
+            pos_src, pos_dst, data = self.select(x, y, i)
+            xs0, xs1 = pos_src[0]
+            ys0, ys1 = pos_src[1]
+            xd0, xd1 = pos_dst[0]
+            yd0, yd1 = pos_dst[1]
+            data[ys0:ys1, xs0:xs1, :] = self.buffer[yd0:yd1, xd0:xd1, :]
+        self.buffer_dirty = False
+
+    def select(self, x, y, idx):
+        w, h = self.width, self.height
+        lst = [(0, 0), (0, h), (w, 0), (w, h)]
+        if idx == 0:
+            x0, y0 = x % self.patch_size, y % self.patch_size
+            x1 = min(x0 + w, self.patch_size)
+            y1 = min(y0 + h, self.patch_size)
+        elif idx == 1:
+            y += h
+            x0, y0 = x % self.patch_size, y % self.patch_size
+            x1 = min(x0 + w, self.patch_size)
+            y1 = max(y0 - h, 0)
+        elif idx == 2:
+            x += w
+            x0, y0 = x % self.patch_size, y % self.patch_size
+            x1 = max(x0 - w, 0)
+            y1 = min(y0 + h, self.patch_size)
+        else:
+            x += w
+            y += h
+            x0, y0 = x % self.patch_size, y % self.patch_size
+            x1 = max(x0 - w, 0)
+            y1 = max(y0 - h, 0)
+        xi, yi = x // self.patch_size, y // self.patch_size
+        cur = self.data.setdefault(
+            (xi, yi), np.zeros((self.patch_size, self.patch_size, 4), dtype=np.uint8)
+        )
+        x0_img, y0_img = lst[idx]
+        x1_img = x0_img + x1 - x0
+        y1_img = y0_img + y1 - y0
+        sort = lambda a, b: ((a, b) if a < b else (b, a))
+        return (
+            (sort(x0, x1), sort(y0, y1)),
+            (sort(x0_img, x1_img), sort(y0_img, y1_img)),
+            cur,
+        )
+
+    def draw_buffer(self):
+        self.canvas[1].clear()
+        self.canvas[1].put_image_data(self.buffer, 0, 0)
+
+    def fill_selection(self, img):
+        self.sel_buffer = img
+        self.sel_dirty = True
+
+    def draw_selection_box(self):
+        x0, y0 = self.cursor
+        size = self.selection_size
+        if self.sel_dirty:
+            self.canvas[2].clear()
+            self.canvas[2].put_image_data(self.sel_buffer, x0, y0)
+        self.canvas[-1].clear()
+        self.canvas[-1].stroke_style = "#0a0a0a"
+        self.canvas[-1].stroke_rect(x0, y0, size, size)
+        self.canvas[-1].stroke_style = "#ffffff"
+        self.canvas[-1].stroke_rect(x0 - 1, y0 - 1, size + 2, size + 2)
+        self.canvas[-1].stroke_style = "#000000"
+        self.canvas[-1].stroke_rect(x0 - 2, y0 - 2, size + 4, size + 4)
+
+    def write_selection_to_buffer(self):
+        x0, y0 = self.cursor
+        x1, y1 = x0 + self.selection_size, y0 + self.selection_size
+        self.buffer[y0:y1, x0:x1] = self.sel_buffer
+        self.sel_dirty = False
+        self.sel_buffer = self.sel_buffer_bak.copy()
+        self.buffer_dirty = True
+        self.buffer_updated = True
+
+    def read_selection_from_buffer(self):
+        x0, y0 = self.cursor
+        x1, y1 = x0 + self.selection_size, y0 + self.selection_size
+        self.sel_buffer = self.buffer[y0:y1, x0:x1]
+        self.sel_dirty = False
+
+    def base64_to_numpy(self, base64_str):
+        try:
+            data = base64.b64decode(str(base64_str))
+            pil = Image.open(io.BytesIO(data))
+            arr = np.array(pil)
+            ret = arr
+        except:
+            ret = np.tile(
+                np.array([255, 0, 0, 255], dtype=np.uint8),
+                (self.selection_size, self.selection_size, 1),
+            )
+        return ret
+
+    def numpy_to_base64(self, arr):
+        out_pil = Image.fromarray(arr)
+        out_buffer = io.BytesIO()
+        out_pil.save(out_buffer, format="PNG")
+        out_buffer.seek(0)
+        base64_bytes = base64.b64encode(out_buffer.read())
+        base64_str = base64_bytes.decode("ascii")
+        return base64_str
+
+    def export(self):
+        if self.sel_dirty:
+            self.write_selection_to_buffer()
+        if self.buffer_dirty:
+            self.buffer2data()
+        xmin, xmax, ymin, ymax = 0, 0, 0, 0
+        if len(self.data.keys()) == 0:
+            return np.zeros(
+                (self.selection_size, self.selection_size, 4), dtype=np.uint8
+            )
+        for xi, yi in self.data.keys():
+            buf = self.data[(xi, yi)]
+            if buf.sum() > 0:
+                xmin = min(xi, xmin)
+                xmax = max(xi, xmax)
+                ymin = min(yi, ymin)
+                ymax = max(yi, ymax)
+        yn = ymax - ymin + 1
+        xn = xmax - xmin + 1
+        image = np.zeros(
+            (yn * self.patch_size, xn * self.patch_size, 4), dtype=np.uint8
+        )
+        for xi, yi in self.data.keys():
+            buf = self.data[(xi, yi)]
+            if buf.sum() > 0:
+                y0 = (yi - ymin) * self.patch_size
+                x0 = (xi - xmin) * self.patch_size
+                image[y0 : y0 + self.patch_size, x0 : x0 + self.patch_size] = buf
+        ylst, xlst = image[:, :, -1].nonzero()
+        if len(ylst) > 0:
+            yt, xt = ylst.min(), xlst.min()
+            yb, xb = ylst.max(), xlst.max()
+            image = image[yt : yb + 1, xt : xb + 1]
+            return image
+        else:
+            return np.zeros(
+                (self.selection_size, self.selection_size, 4), dtype=np.uint8
+            )

js/mode.js

@@ -0,0 +1,6 @@
+function(mode){
+    let app=document.querySelector("gradio-app").shadowRoot;
+    let frame=app.querySelector("#sdinfframe").contentWindow.document;
+    frame.querySelector("#mode").value=mode;
+    return mode;
+}

js/outpaint.js

@@ -0,0 +1,24 @@
+function(a){
+    if(!window.my_observe_outpaint)
+    {
+        console.log("setup outpaint here");
+        window.my_observe_outpaint = new MutationObserver(function (event) {
+            console.log(event);
+            let app=document.querySelector("gradio-app").shadowRoot;
+            let frame=app.querySelector("#sdinfframe").contentWindow.document;
+            frame.querySelector("#outpaint").click();
+        });
+        window.my_observe_outpaint_target=document.querySelector("gradio-app").shadowRoot.querySelector("#output span")
+        window.my_observe_outpaint.observe(window.my_observe_outpaint_target, {
+            attributes: false, 
+            subtree: true,
+            childList: true, 
+            characterData: true
+        });
+    }
+    let app=document.querySelector("gradio-app").shadowRoot;
+    let frame=app.querySelector("#sdinfframe").contentWindow.document;
+    let button=frame.querySelector("#transfer");
+    button.click();
+    return a;
+}

js/proceed.js

@@ -0,0 +1,22 @@
+function(sel_buffer_str,
+    prompt_text,
+    strength,
+    guidance,
+    step,
+    resize_check,
+    fill_mode,
+    enable_safety,
+    state){
+    sel_buffer = document.querySelector("gradio-app").shadowRoot.querySelector("#input textarea").value;
+    return [
+        sel_buffer,
+        prompt_text,
+        strength,
+        guidance,
+        step,
+        resize_check,
+        fill_mode,
+        enable_safety,
+        state
+    ]
+}

js/setup.js

@@ -0,0 +1,22 @@
+function(token_val, width, height, size){
+    let app=document.querySelector("gradio-app").shadowRoot;
+    app.querySelector("#sdinfframe").style.height=height+"px";
+    let frame=app.querySelector("#sdinfframe").contentWindow.document;
+    if(frame.querySelector("#setup").value=="0")
+    {
+        window.my_setup=setInterval(function(){
+            let frame=document.querySelector("gradio-app").shadowRoot.querySelector("#sdinfframe").contentWindow.document;
+            console.log("Check PyScript...")
+            if(frame.querySelector("#setup").value=="1")
+            {
+                frame.querySelector("#draw").click();
+                clearInterval(window.my_setup);
+            }
+        },100)
+    }
+    else
+    {
+        frame.querySelector("#draw").click();
+    }
+    return [token_val, width, height, size];
+}

js/upload.js

@@ -0,0 +1,19 @@
+function(a,b){
+    if(!window.my_observe_upload)
+    {
+        console.log("setup upload here");
+        window.my_observe_upload = new MutationObserver(function (event) {
+            console.log(event);
+            var frame=document.querySelector("gradio-app").shadowRoot.querySelector("#sdinfframe").contentWindow.document;
+            frame.querySelector("#upload").click();
+        });
+        window.my_observe_upload_target = document.querySelector("gradio-app").shadowRoot.querySelector("#upload span");
+        window.my_observe_upload.observe(window.my_observe_upload_target, {
+            attributes: false, 
+            subtree: true,
+            childList: true, 
+            characterData: true
+        });
+    }
+    return [a,b];
+}

packages.txt

@@ -0,0 +1,2 @@
+build-essential
+libopencv-dev

perlin2d.py

@@ -0,0 +1,45 @@
+import numpy as np
+
+##########
+# https://stackoverflow.com/questions/42147776/producing-2d-perlin-noise-with-numpy/42154921#42154921
+def perlin(x, y, seed=0):
+    # permutation table
+    np.random.seed(seed)
+    p = np.arange(256, dtype=int)
+    np.random.shuffle(p)
+    p = np.stack([p, p]).flatten()
+    # coordinates of the top-left
+    xi, yi = x.astype(int), y.astype(int)
+    # internal coordinates
+    xf, yf = x - xi, y - yi
+    # fade factors
+    u, v = fade(xf), fade(yf)
+    # noise components
+    n00 = gradient(p[p[xi] + yi], xf, yf)
+    n01 = gradient(p[p[xi] + yi + 1], xf, yf - 1)
+    n11 = gradient(p[p[xi + 1] + yi + 1], xf - 1, yf - 1)
+    n10 = gradient(p[p[xi + 1] + yi], xf - 1, yf)
+    # combine noises
+    x1 = lerp(n00, n10, u)
+    x2 = lerp(n01, n11, u)  # FIX1: I was using n10 instead of n01
+    return lerp(x1, x2, v)  # FIX2: I also had to reverse x1 and x2 here
+
+
+def lerp(a, b, x):
+    "linear interpolation"
+    return a + x * (b - a)
+
+
+def fade(t):
+    "6t^5 - 15t^4 + 10t^3"
+    return 6 * t ** 5 - 15 * t ** 4 + 10 * t ** 3
+
+
+def gradient(h, x, y):
+    "grad converts h to the right gradient vector and return the dot product with (x,y)"
+    vectors = np.array([[0, 1], [0, -1], [1, 0], [-1, 0]])
+    g = vectors[h % 4]
+    return g[:, :, 0] * x + g[:, :, 1] * y
+
+
+##########

requirements.txt

@@ -0,0 +1,12 @@
+--extra-index-url https://download.pytorch.org/whl/cu113
+imageio==2.19.5
+imageio-ffmpeg==0.4.7
+numpy==1.22.4
+opencv-python-headless==4.6.0.66
+torch==1.12.0+cu113
+torchvision==0.13.0+cu113
+scipy
+scikit-image
+diffusers==0.3.0
+transformers
+ftfy

utils.py

@@ -0,0 +1,154 @@
+from PIL import Image
+from PIL import ImageFilter
+import cv2
+import numpy as np
+import scipy
+import scipy.signal
+from scipy.spatial import cKDTree
+
+import os
+from perlin2d import *
+
+patch_match_compiled = True
+if os.name != "nt":
+    try:
+        from PyPatchMatch import patch_match
+    except Exception as e:
+        try:
+            import patch_match
+        except Exception as e:
+            patch_match_compiled = False
+
+try:
+    patch_match
+except NameError:
+    print("patch_match compiling failed")
+    patch_match_compiled = False
+
+
+
+
+def edge_pad(img, mask, mode=1):
+    if mode == 0:
+        nmask = mask.copy()
+        nmask[nmask > 0] = 1
+        res0 = 1 - nmask
+        res1 = nmask
+        p0 = np.stack(res0.nonzero(), axis=0).transpose()
+        p1 = np.stack(res1.nonzero(), axis=0).transpose()
+        min_dists, min_dist_idx = cKDTree(p1).query(p0, 1)
+        loc = p1[min_dist_idx]
+        for (a, b), (c, d) in zip(p0, loc):
+            img[a, b] = img[c, d]
+    elif mode == 1:
+        record = {}
+        kernel = [[1] * 3 for _ in range(3)]
+        nmask = mask.copy()
+        nmask[nmask > 0] = 1
+        res = scipy.signal.convolve2d(
+            nmask, kernel, mode="same", boundary="fill", fillvalue=1
+        )
+        res[nmask < 1] = 0
+        res[res == 9] = 0
+        res[res > 0] = 1
+        ylst, xlst = res.nonzero()
+        queue = [(y, x) for y, x in zip(ylst, xlst)]
+        # bfs here
+        cnt = res.astype(np.float32)
+        acc = img.astype(np.float32)
+        step = 1
+        h = acc.shape[0]
+        w = acc.shape[1]
+        offset = [(1, 0), (-1, 0), (0, 1), (0, -1)]
+        while queue:
+            target = []
+            for y, x in queue:
+                val = acc[y][x]
+                for yo, xo in offset:
+                    yn = y + yo
+                    xn = x + xo
+                    if 0 <= yn < h and 0 <= xn < w and nmask[yn][xn] < 1:
+                        if record.get((yn, xn), step) == step:
+                            acc[yn][xn] = acc[yn][xn] * cnt[yn][xn] + val
+                            cnt[yn][xn] += 1
+                            acc[yn][xn] /= cnt[yn][xn]
+                            if (yn, xn) not in record:
+                                record[(yn, xn)] = step
+                                target.append((yn, xn))
+            step += 1
+            queue = target
+        img = acc.astype(np.uint8)
+    else:
+        nmask = mask.copy()
+        ylst, xlst = nmask.nonzero()
+        yt, xt = ylst.min(), xlst.min()
+        yb, xb = ylst.max(), xlst.max()
+        content = img[yt : yb + 1, xt : xb + 1]
+        img = np.pad(
+            content,
+            ((yt, mask.shape[0] - yb - 1), (xt, mask.shape[1] - xb - 1), (0, 0)),
+            mode="edge",
+        )
+    return img, mask
+
+
+def perlin_noise(img, mask):
+    lin = np.linspace(0, 5, mask.shape[0], endpoint=False)
+    x, y = np.meshgrid(lin, lin)
+    avg = img.mean(axis=0).mean(axis=0)
+    # noise=[((perlin(x, y)+1)*128+avg[i]).astype(np.uint8) for i in range(3)]
+    noise = [((perlin(x, y) + 1) * 0.5 * 255).astype(np.uint8) for i in range(3)]
+    noise = np.stack(noise, axis=-1)
+    # mask=skimage.measure.block_reduce(mask,(8,8),np.min)
+    # mask=mask.repeat(8, axis=0).repeat(8, axis=1)
+    # mask_image=Image.fromarray(mask)
+    # mask_image=mask_image.filter(ImageFilter.GaussianBlur(radius = 4))
+    # mask=np.array(mask_image)
+    nmask = mask.copy()
+    # nmask=nmask/255.0
+    nmask[mask > 0] = 1
+    img = nmask[:, :, np.newaxis] * img + (1 - nmask[:, :, np.newaxis]) * noise
+    # img=img.astype(np.uint8)
+    return img, mask
+
+
+def gaussian_noise(img, mask):
+    noise = np.random.randn(mask.shape[0], mask.shape[1], 3)
+    noise = (noise + 1) / 2 * 255
+    noise = noise.astype(np.uint8)
+    nmask = mask.copy()
+    nmask[mask > 0] = 1
+    img = nmask[:, :, np.newaxis] * img + (1 - nmask[:, :, np.newaxis]) * noise
+    return img, mask
+
+
+def cv2_telea(img, mask):
+    ret = cv2.inpaint(img, 255 - mask, 5, cv2.INPAINT_TELEA)
+    return ret, mask
+
+
+def cv2_ns(img, mask):
+    ret = cv2.inpaint(img, 255 - mask, 5, cv2.INPAINT_NS)
+    return ret, mask
+
+
+def patch_match_func(img, mask):
+    ret = patch_match.inpaint(img, mask=255 - mask, patch_size=3)
+    return ret, mask
+
+
+def mean_fill(img, mask):
+    avg = img.mean(axis=0).mean(axis=0)
+    img[mask < 1] = avg
+    return img, mask
+
+
+functbl = {
+    "gaussian": gaussian_noise,
+    "perlin": perlin_noise,
+    "edge_pad": edge_pad,
+    "patchmatch": patch_match_func if (os.name != "nt" and patch_match_compiled) else edge_pad,
+    "cv2_ns": cv2_ns,
+    "cv2_telea": cv2_telea,
+    "mean_fill": mean_fill,
+}