Proof. By contradiction, assume that $(u,v) \not \equiv (0,0)$, then we can derive that $u>0$ and $v>0$ in $\mathbb {R}^n_+$. Indeed, if there exists some $\hat {x}\in \mathbb {R}_+^n$ such that $u(\hat {x})=0$, from the anti-symmetry of $u$, we have

\[ (-\Delta)^{s_1}u(\hat{x})=\int_{\mathbb{R}^n}\frac{-u(y)}{|\hat{x}-y|^{n+2s_1}}\,{\rm d}y<0, \]

which contradicts with the equation

\[ (-\Delta)^{s_1}u(\hat{x})=|\hat{x}|^\alpha v^p(\hat{x})\geq 0. \]

Thus $u(x)>0,$ and using the same arguments as above, we easily obtain $v(x)>0$. Therefore, we may assume that $u(x)>0$ and $v(x)>0$ in the rest proof of (i) of theorem 1.

Let $u_\lambda (x)$ and $v_\lambda (x)$ be the Kelvin transform of $u(x)$ and $v(x)$ centred at origin, respectively

\begin{align*} u_\lambda(x)& =\left(\frac{\lambda}{|x|}\right)^{n-2s_1}u\left(\frac{\lambda^2x}{|x|^2}\right),\\ v_\lambda(x)& =\left(\frac{\lambda}{|x|}\right)^{n-2s_2}v\left(\frac{\lambda^2x}{|x|^2}\right) \end{align*}

for arbitrary $x\in \mathbb {R}^n\setminus \{0\}$. By an elementary calculation, $u_\lambda (x)$ and $v_\lambda (x)$ satisfy the following system:

\[ \left\{\begin{array}{@{}ll} (-\Delta)^{s_1}u_\lambda(x)=|x|^{\alpha}\left(\dfrac{\lambda}{|x|}\right)^{\tau_1}v_\lambda^p(x), & x\in \mathbb{R}_+^n, \\ (-\Delta)^{s_2}v_\lambda(x)=|x|^{\beta}\left(\dfrac{\lambda}{|x|}\right)^{\tau_2}u_\lambda^q(x), & x\in \mathbb{R}_+^n, \end{array}\right. \]

where

\[ \tau_1=n+2s_1+2\alpha-p(n-2s_2)\quad\text{and}\quad \tau_2=n+2s_2+2\beta -q(n-2s_1). \]

Note that both $\tau _1$ and $\tau _2$ are nonnegative and they will not be zero simultaneously, since $(p,q)\in \mathcal {R}_{sub}$.

Denote

\[ U_\lambda(x)=u_\lambda(x)-u(x)\quad \text{and}\quad V_\lambda(x)=v_\lambda(x)-v(x). \]

By elementary calculations and the mean value theorem, for $x\in B_\lambda ^+$, there holds

(3.1)\begin{align} (-\Delta)^{s_1}U_\lambda(x)& =|x|^{\alpha}\left(\frac{\lambda}{|x|}\right)^{\tau_1}v_\lambda^p(x)-|x|^{\alpha}v^p(x) \nonumber\\ & =|x|^\alpha\left[(v_\lambda^p(x)-v^p(x))+\left(\left(\frac{\lambda}{|x|}\right)^{\tau_1}-1\right)v_\lambda^p(x)\right]\nonumber\\& \geq|x|^\alpha p\xi_{\lambda}^{p-1}(x)V_\lambda(x), \end{align}

(3.2)\begin{align} (-\Delta)^{s_2}V_\lambda(x)& \geq|x|^\beta q\eta_{\lambda}^{q-1}(x)U_\lambda(x), \end{align}

where $\xi _\lambda (x)$ is between $v(x)$ and $v_\lambda (x)$, $\eta _\lambda (x)$ is between $u(x)$ and $u_\lambda (x)$. Note that

(3.3)\begin{equation} U_\lambda(x)={-}\left(\frac{\lambda}{|x|}\right)^{n-2s_1}U_\lambda(x^\lambda)\quad \text{and}\quad V_\lambda(x)={-}\left(\frac{\lambda}{|x|}\right)^{n-2s_2}V_\lambda(x^\lambda). \end{equation}

Next, we will use the method of moving spheres to claim that $U_\lambda (x)\geq 0$ and $V_\lambda (x)\geq 0$ in $B_\lambda ^+$ for any $\lambda >0$.

Step 1. Give a start point. We show that for sufficiently small $\lambda >0$,

(3.4)\begin{equation} U_\lambda(x)\geq 0\ \text{and}\ V_\lambda(x)\geq 0,\quad x\in B_\lambda^+. \end{equation}

Suppose (3.4) is not true, there must exist a point $\bar {x}\in B_\lambda ^+$ such that at least one of $U_\lambda (\bar {x})$ and $V_\lambda (\bar {x})$ is negative at this point. Without loss of generality, we assume

\[ U_\lambda(\bar{x})=\inf_{x\in B_\lambda^+}\{U_\lambda(x),V_\lambda(x)\}<0. \]

We will obtain contradictions for all four possible cases, respectively.

Case 1. $(p,q)\in \mathcal {R}_{sub}$ and $p\geq 1, q\geq 1$. Due to $p\geq 1$, by the convexity of the function $f(t)=t^p$, then we can take $\xi _\lambda (x)=v(x)$ in (3.1). From equation (3.1) and lemma 2.2, we have

(3.5)\begin{align} |\bar{x}|^\alpha pv^{p-1}(\bar{x})U_\lambda(\bar{x})& \leq|\bar{x}|^\alpha pv^{p-1}(\bar{x})V_\lambda(\bar{x})\leq(-\Delta)^{s_1}U_\lambda(\bar{x})\nonumber\\ & \leq C(n,s_1)U_\lambda(\bar{x})\left((\lambda-|\bar{x}|)^{{-}2s_1}+\frac{\delta^n}{\bar{x}_n^{n+2s_1}}\right). \end{align}

Hence,

(3.6)\begin{equation} v^{p-1}(\bar{x})\geq\frac{C(n,s_1)\left((\lambda-|\bar{x}|)^{{-}2s_1}+\frac{\delta^n}{\bar{x}_n^{n+2s_1}}\right)}{p|\bar{x}|^\alpha }. \end{equation}

If $\delta =\min \{\lambda -|\bar {x}|,\bar {x}_n\}=\lambda -|\bar {x}|$, which implies that $\lambda -|\bar {x}|\leq \bar {x}_n\leq |\bar {x}|$, using the fact and (3.6), we obtain

(3.7)\begin{equation} v^{p-1}(\bar{x})\geq\frac{C(\lambda-|\bar{x}|)^{{-}2s_1}}{p|\bar{x}|^\alpha }\geq C|\bar{x}|^{{-}2s_1-\alpha}. \end{equation}

As $\lambda \to 0$, the right-hand side of (3.7) will go to infinity since $\alpha >-2s_1$. This is impossible.

If $\delta =\min \{\lambda -|\bar {x}|,\bar {x}_n\}=\bar {x}_n$, from $\bar {x}_n\leq |\bar {x}|$ and (3.6), we derive

(3.8)\begin{equation} v^{p-1}(\bar{x})\geq\frac{C\bar{x}_n^{{-}2s_1}}{p|\bar{x}|^\alpha }\geq C|\bar{x}|^{{-}2s_1-\alpha}, \end{equation}

which is also impossible.

Case 2. $0< p,q<1$. Due to $p<1$, we can take $\xi _\lambda (x) =v_\lambda (x)$. From equation (3.1) and lemma 2.2, we have

(3.9)\begin{align} |\bar{x}|^\alpha p v_\lambda^{p-1}(\bar{x})U_\lambda(\bar{x})& \leq|\bar{x}|^\alpha p v_\lambda^{p-1}(\bar{x})V_\lambda(\bar{x})\leq (-\Delta)^{s_1}U_\lambda(\bar{x}) \nonumber\\ & \leq C(n,s_1)U_\lambda(\bar{x})\left((\lambda-|\bar{x}|)^{{-}2s_1}+\frac{\delta^n}{\bar{x}_n^{n+2s_1}}\right). \end{align}

Analogous to (3.7) and (3.8), there holds

(3.10)\begin{equation} v_\lambda^{p-1}(\bar{x})\geq\frac{C\bar{x}_n^{{-}2s_1}}{|\bar{x}|^\alpha}, \end{equation}

or

(3.11)\begin{equation} v_\lambda^{p-1}(\bar{x})\geq\frac{C(\lambda-|\bar{x}|)^{{-}2s_1}}{|\bar{x}|^\alpha }. \end{equation}

Applying lemma 2.3 and the mean value theorem, we obtain that for $x\in B_1^+$,

\begin{align*} u(x)& \geq C\int_{\mathbb{R}_+^n}\left(\frac{1}{|x-y|^{n-2s_1}}-\frac{1}{|x^*-y|^{n-2s_1}}\right)|y|^{\alpha}v^p(y)\,{\rm d}y \\ & \geq C\displaystyle\int_{B_{1}(2e_n)}\left(\displaystyle\frac{1}{|x-y|^{n-2s_1}}-\frac{1}{|x^*-y|^{n-2s_1}}\right){\rm d}y \\ & \geq C\int_{B_{1}(2e_n)}\frac{x_ny_n}{|x^*-y|^{n-2s_1+2}}\,{\rm d}y \\ & \geq Cx_n. \end{align*}

For $x\in (B_1^+)^C$, we derive

\begin{align*} u(x)& \geq C\int_{\mathbb{R}_+^n}\left(\frac{1}{|x-y|^{n-2s_1}}-\frac{1}{|x^*-y|^{n-2s_1}}\right)|y|^{\alpha}v^p(y)\,{\rm d}y \\ & \geq C\int_{B_{1}(2e_n)}\frac{x_ny_n}{|x^*-y|^{n-2s_1+2}}\,{\rm d}y \\ & \geq C\frac{x_n}{|x|^{n-2s_1+2}}. \end{align*}

Similarly, we have

\[ v(x)\geq\left\{\begin{array}{@{}ll} Cx_n, & x\in B_1^+, \\ C\dfrac{x_n}{|x|^{n-2s_2+2}}, & x\in (B_1^+)^C. \end{array}\right. \]

Then by the definition of $v_\lambda (x)$, we obtain

(3.12)\begin{equation} v_\lambda(x)\geq \left\{\begin{array}{@{}ll} C\left(\dfrac{\lambda}{|x|}\right)^{n-2s_2+2}x_n, & x^\lambda\in B_1^+, \\ C\dfrac{x_n}{\lambda^{n-2s_2+2}}, & x^\lambda\in (B_1^+)^C. \end{array}\right. \end{equation}

If $\delta =\lambda -|\bar {x}|\leq \bar {x}_n$, that is, $|\bar {x}|\geq \frac {\lambda }{2}$, combining (3.11) and (3.12), we conclude that for $\bar {x}\in B_\lambda ^+$ and sufficiently small $\lambda$,

\[ (\lambda-|\bar{x}|)^{{-}2s_1} \leq C\bar{x}_n^{p-1}|\bar{x}|^\alpha\leq C(\lambda-|\bar{x}|)^{p-1}|\bar{x}|^\alpha, \]

which gives that

(3.13)\begin{equation} \left(\frac{\lambda}{|\bar{x}|}-1\right)^{{-}2s_1-p+1}\leq C|\bar{x}|^{p+2s_1+\alpha-1}. \end{equation}

Due to $\min \{p+2s_1, p+2s_1+\alpha \}>1$ and $\frac {\lambda }{|\bar {x}|}\leq 2$, inequality (3.13) is impossible as $\lambda >0$ sufficiently small.

If $\delta =\bar {x}_n$, it follows from (3.10) and (3.12) that

\[ \frac{C\bar{x}_n^{{-}2s_1}}{|\bar{x}|^\alpha }\leq v_\lambda^{p-1}(\bar{x})\leq C \bar{x}_n^{p-1}, \]

which implies that

\[ |\bar{x}|^{{-}p-2s_1-\alpha+1}\leq C\quad \text{if}\ \alpha\geq 0,\quad \text{and}\quad \bar{x}_n^{{-}p-2s_1-\alpha+1}\leq C\quad \text{if}\ \alpha< 0. \]

Either one of the two inequalities will yield a contradiction since the left terms go to infinity as $\min \{p+2s_1,p+2s_1+\alpha \}>1$ and $\lambda >0$ small enough.

For case 3: $(p,q)\in \mathcal {R}_{sub}, p\geq 1, 0< q<1$ and case 4: $(p,q)\in \mathcal {R}_{sub}, 0< p<1, q\geq 1$, similar argument as that of cases 1 and 2 can show that $U_\lambda (x)\geq 0$ and $V_\lambda (x)\geq 0$ in $B_\lambda ^+$ for sufficiently small $\lambda >0$. Therefore, (3.4) holds.

Step 2. Now we move the sphere $S_\lambda$ outwards as long as (3.4) holds. Define

\[ \lambda_0=\sup\{\lambda|\,U_\mu(x)\geq 0, V_\mu(x)\geq 0,\ x\in B_\mu^+,\ \forall\ 0<\mu<\lambda\}. \]

We will show that $\lambda _0=+\infty$. Suppose on the contrary that $0<\lambda _0<+\infty$. We want to show that there exists some small $\varepsilon >0$ such that for any $\lambda \in (\lambda _0,\lambda _0+\varepsilon )$,

(3.14)\begin{equation} U_\lambda(x)\geq 0\ \text{and}\ V_\lambda(x)\geq 0,\quad x\in B_\lambda^+. \end{equation}

This implies that the plane $S_{\lambda _0}$ will be moved outwards a little bit further, which contradicts with the definition of $\lambda _0$.

Firstly, we claim that

(3.15)\begin{equation} U_{\lambda_0}(x)> 0\ \text{and}\ V_{\lambda_0}(x)> 0,\quad x\in B_{\lambda_0}^+. \end{equation}

Indeed, if there exists some point $x^0\in B_{\lambda _0}^+$ such that $U_{\lambda _0}(x^0)= 0$, we have

(3.16)\begin{equation} (-\Delta)^{s_1}U_{\lambda_0}(x^0)=C\int_{\mathbb{R}^n}\frac{-U_{\lambda_0}(y)}{|x^0-y|^{n+2s_1}}\,{\rm d}y\leq 0. \end{equation}

On the other hand, it is easy to get that

(3.17)\begin{align} (-\Delta)^{s_1}U_{\lambda_0}(x^0)& =|x^0|^{\alpha}\left(\frac{\lambda_0}{|x^0|}\right)^{\tau_1}v_{\lambda_0}^p(x^0)-|x^0|^{\alpha}v^p(x^0) \nonumber\\ & =|x^0|^\alpha \left(\left(\frac{\lambda_0}{|x^0|}\right)^{\tau_1}-1\right)v_\lambda^p(x^0)+p|x^0|^\alpha \xi_\lambda^{p-1}(x^0)V_{\lambda_0}(x^0). \end{align}

If $\tau _1>0$, then $(-\Delta )^{s_1}U_{\lambda _0}(x^0)>0$, where we use the facts that $V_{\lambda _0}\geq 0$ and $v>0$ in $\mathbb {R}^n_+$. If $\tau _1=0$, then we have that $\tau _2>0$. Moreover, $V_{\lambda _0}(x^0)=0$ follows from (3.16) and (3.17). Using an argument similar to (3.16) and (3.17), we derive

\begin{align*} 0\geq(-\Delta)^{s_2}V_{\lambda_0}(x^0)& = |x^0|^\beta \left(\left(\frac{\lambda_0}{|x^0|}\right)^{\tau_2}-1\right)u_\lambda^q(x^0)+|x^0|^\beta \eta_\lambda^{q-1}(x^0)U_{\lambda_0}(x^0) \\ & = |x^0|^\beta \left(\left(\frac{\lambda_0}{|x^0|}\right)^{\tau_2}-1\right)u_\lambda^q(x^0)>0, \end{align*}

which is absurd. Thus, $U_{\lambda _0}(x)> 0$ is proved. Similarly, we derive that $V_{\lambda _0}(x)> 0$. Hence, (3.15) holds.

Next, we will show that the sphere can be moved further outwards. The continuity of $u(x)$ and (3.15) yield that there exists some sufficiently small $l\in (0,\frac {\lambda _0}{2})$ and $\varepsilon _1\in (0,\frac {\lambda _0}{2})$ such that for $\lambda \in (\lambda _0,\lambda _0+\varepsilon _1)$,

(3.18)\begin{equation} U_\lambda(x)\geq 0, \quad x\in B_{\lambda_0-l}^+. \end{equation}

For $x\in B_\lambda ^+\setminus B_{\lambda _0-l}^+$, using the similar proof of (3.4), we can deduce that

(3.19)\begin{equation} U_\lambda(x)\geq 0,\quad x\in B_\lambda^+{\setminus} B_{\lambda_0-l}^+ . \end{equation}

Note that the distance between $\bar {x}$ and $S_\lambda$, i.e. $\lambda -|\bar {x}|$, plays an important role in this process.

Hence, it follows from (3.18) and (3.19) that for all $\lambda \in (\lambda _0,\lambda _0+\varepsilon _1)$,

\[ U_\lambda(x)\geq 0,\quad x\in B_\lambda^+. \]

Similarly, we can also prove that there exists $\varepsilon _2>0$ such that for all $\lambda \in (\lambda _0, \lambda _0+\varepsilon _2)$,

\[ V_\lambda(x)\geq 0,\quad x\in B_\lambda^+. \]

Let $\varepsilon =\min \{\varepsilon _1,\varepsilon _2\}$, therefore, (3.14) can be completely concluded. This contradicts with the definition of $\lambda _0$. So $\lambda _0=+\infty$.

Then, we have for every $\lambda >0$,

\[ U_\lambda(x)\geq 0,\quad V_\lambda(x)\geq 0,\quad x\in B_\lambda^+, \]

which gives that,

(3.20)\begin{equation} u(x)\geq\left(\frac{\lambda}{|x|}\right)^{n-2s_1}u\left(\frac{\lambda^2x}{|x|^2}\right),\quad \forall \,|x|\geq\lambda, \ \ x\in\mathbb{R}_+^n,\ \forall\, 0<\lambda<{+}\infty. \end{equation}

(3.21)\begin{equation} v(x)\geq\left(\frac{\lambda}{|x|}\right)^{n-2s_2}v\left(\frac{\lambda^2x}{|x|^2}\right),\quad \forall \,|x|\geq\lambda,\ x\in\mathbb{R}_+^n, \ \forall\, 0<\lambda<{+}\infty. \end{equation}

For any given $|x|\geq 1$, let $\lambda =\sqrt {|x|}$, then it follows from (3.20) that

(3.22)\begin{equation} u(x)\geq\left(\min_{x\in S_1^+}u(x)\right)\frac{1}{|x|^{\frac{n-2s_1}{2}}}:=\frac{C}{|x|^{\frac{n-2s_1}{2}}}\geq\frac{Cx_n}{|x|^{\frac{n-2s_1}{2}+1}}, \end{equation}

and similarly from (3.21), we obtain

(3.23)\begin{equation} v(x)\geq\frac{Cx_n}{|x|^{\frac{n-2s_2}{2}+1}}. \end{equation}

Now we make full use of the above properties to derive some lower-bound estimates of solutions to (1.1) through iteration technique.

Let $\theta _0=\frac {n-2s_1}{2}+1$, $\sigma _0=\frac {n-2s_2}{2}+1$. From lemma 2.3, inequality (3.23) and the mean value theorem, we have for $x_n>1$,

(3.24)\begin{align} u(x)& \geq C\int_{\mathbb{R}_+^n}\left(\frac{1}{|x-y|^{n-2s_1}}-\frac{1}{|x^*-y|^{n-2s_1}}\right)|y|^{\alpha}v^p(y)\,{\rm d}y\nonumber\\ & \geq C\int_{2|x|}^{4|x|}\int_{2|x|\leq |y'|\leq 4|x|}\frac{x_ny_n}{|x^*-y|^{n-2s_1+2}}\frac{y_n^p}{|y|^{p\sigma_0-\alpha}}\,{\rm d}y'\,{\rm d}y_n\nonumber\\ & \geq C\frac{x_n}{|x|^{(n-2s_1+2)+(p\sigma_0-\alpha)}}\int_{2|x|}^{4|x|}\int_{2|x|\leq |y'|\leq 4|x|}y_n^{p+1}\,{\rm d}y'\,{\rm d}y_n\nonumber\\ & \geq C\frac{x_n}{|x|^{p\sigma_0-\alpha-2s_1+3-(p+2)}}. \end{align}

Similarly, we have

(3.25)\begin{equation} v(x) \geq C\frac{x_n}{|x|^{q(\theta_0-1)-(\beta+2s_2)+1}} \end{equation}

Denote $\theta _1=p(\sigma _0-1)-(\alpha +2 s_1)+1$ and $\sigma _1=q(\theta _0-1)-(\beta +2s_2)+1$. Repeat the above process replacing (3.23) by (3.25), then we have

\[ u(x)\geq C\int_{2|x|}^{4|x|}\int_{2|x|\leq |y'|\leq 4|x| }\frac{x_ny_n}{|x^*-y|^{n-2s_1+2}}\frac{y_n^p}{|y|^{p\sigma_1-\alpha}}\,{\rm d}y'\,{\rm d}y_n\geq C\frac{x_n}{|x|^{\theta_2}}, \]

and analogously,

(3.26)\begin{equation} v(x)\geq C\frac{x_n}{|x|^{\sigma_2}}, \end{equation}

where $\theta _2=p(\sigma _1-1)-\alpha -2s_1+1$ and $\sigma _2=q(\theta _1-1)-\beta -2s_2+1$.

After such $k$ iteration steps, we derive

(3.27)\begin{equation} u(x)\geq C\frac{x_n}{|x|^{\theta_{k+1}}},\quad v(x)\geq C\frac{x_n}{|x|^{\sigma_{k+1}}}, \end{equation}

where $\theta _{k+1}=p(\sigma _{k}-1)-\alpha -2s_1+1$ and $\sigma _{k+1}=q(\theta _{k}-1)-\beta -2s_2+1$. Elementary calculations give that

(3.28)\begin{align} \theta_{2m}& =\frac{n-2s_1}{2}(pq)^{m}-\left[\left(p(2s_2+\beta)+(2s_1+\alpha)\right)\frac{1-(pq)^m}{1-pq}\right]+1,\nonumber\\ \theta_{2m+1}& =\left(\frac{p(n-2s_2)}{2}-2s_1-\alpha \right)(pq)^m\nonumber\\ & \quad-\left[\left(p(2s_2+\beta)+(2s_1+\alpha)\right)\frac{1-(pq)^m}{1-pq}\right]+1,\nonumber\\ \sigma_{2m}& =\frac{n-2s_2}{2}(pq)^{m}-\left[\left(q(2s_1+\alpha)+(2s_2+\beta)\right)\frac{1-(pq)^m}{1-pq}\right]+1, \nonumber\\ \sigma_{2m+1}& =\left(\frac{q(n-2s_1)}{2}-2s_2-\beta \right)(pq)^m\nonumber\\ & \quad-\left[\left(q(2s_1+\alpha)+(2s_2+\beta)\right)\frac{1-(pq)^m}{1-pq}\right]+1, \end{align}

where $m=0,1,2,\ldots$.

For the case $pq\geq 1$, we claim that both $\{\theta _k\}$ and $\{\sigma _k\}$ are decreasing sequences and unbounded from below. Denote

\begin{align*} & A_e=p\frac{n-2s_2}{2}-\frac{n-2s_1}{2}-2s_1-\alpha,\\ & A_o=q\frac{n-2s_1}{2}-\frac{n-2s_2}{2}-2s_2-\beta. \end{align*}

Note that $A_e,A_o\leq 0$ and they will not be zero simultaneously, since $(p,q)\in \mathcal {R}_{sub}$. By elementary calculations, we have

(3.29)\begin{equation} \theta_{k+1}-\theta_{k}=\left\{\begin{array}{@{}ll} p^{\left[\dfrac{k+1}{2}\right]}q^{\left[\dfrac{k}{2}\right]}A_e\leq 0, & k \text{ is even},\\ p^{\left[\dfrac{k+1}{2}\right]}q^{\left[\dfrac{k}{2}\right]}A_o\leq 0, & k \text{ is odd}. \end{array}\right. \end{equation}

Hence, the sequence $\{\theta _k\}$ is decreasing.

Combining the above properties of $A_e, A_o$, the assumption $pq\geq 1$ and (3.29), we deduce that $\theta _k\to -\infty$ as $k\to +\infty$. Similarly, we can show that $\{\sigma _k\}$ is also decreasing and $\sigma _k\to -\infty$. These and (3.27) indicate that $u(x)$ and $v(x)$ do not belong to any $L_{2s}$. Hence, the solutions $(u,v)\equiv (0,0)$ when $pq\geq 1$.

Next, we consider the case $pq<1$. From (3.28), we can conclude that

(3.30)\begin{align} \theta_k\to1-\frac{p(2s_2+\beta)+(2s_1+\alpha)}{1-pq},\quad \sigma_k\to1-\frac{q(2s_1+\alpha)+(2s_2+\beta)}{1-pq}\quad \text{as }k\to \infty. \end{align}

Since $p+2s_1+\alpha >1$ and $q+2s_2+\beta >1$, we have

\[ \frac{p(2s_2+\beta)+(2s_1+\alpha)}{1-pq}-1>0,\quad \frac{q(2s_1+\alpha)+(2s_2+\beta)}{1-pq}-1>0. \]

Hence, from (3.30) and (3.27), we have for $x_n>1$,

(3.31)\begin{equation} u(x)\geq cx_n^{\frac{p(2s_2+\beta)+(2s_1+\alpha)}{1-pq}+o(1)}, \quad v(x)\geq cx_n^{\frac{q(2s_1+\alpha)+(2s_2+\beta)}{1-pq}+o(1)}. \end{equation}

Combining this with lemma 2.3, we have

\begin{align*} u(x)& \geq C\int_{\mathbb{R}_+^n}\left(\frac{1}{|x-y|^{n-2s_1}}-\frac{1}{|x^*-y|^{n-2s_1}}\right)|y|^{\alpha}v^p(y)\,{\rm d}y \\ & \geq C \int_{2x_n}^{+\infty}\int_{\mathbb{R}^{n-1}}\left(\frac{1}{|x-y|^{n-2s_1}}\right.\\& \quad \left.-\frac{1}{|x^*-y|^{n-2s_1}}\right) y_n^{\left(\frac{q(2s_1+\alpha)+(2s_2+\beta)}{1-pq}+o(1)\right)p+\alpha}\,{\rm d}y'\,{\rm d}y_n \\ & \geq C\int_{2x_n}^{+\infty}y_n^{\left(\frac{q(2s_1+\alpha)+(2s_2+\beta)}{1-pq}+o(1)\right)p+\alpha}\,{\rm d}y_n\\ & \quad \times \int_{\mathbb{R}^{n-1}}\frac{x_ny_n}{\left(|x'-y'|^2+|x_n+y_n|^2\right)^{\frac{n-2s_1+2}{2}}}\,{\rm d}y'. \end{align*}

Let $x'-y'=(x_n+y_n)z'$, we have

\begin{align*} u(x)& \geq C\int_{2x_n}^{+\infty}\frac{y_n^{\left(\frac{q(2s_1+\alpha)+(2s_2+\beta)}{1-pq}+o(1)\right)p+\alpha+1}x_n}{(x_n+y_n)^{3-2s_1}}\,{\rm d}y_n\int_{\mathbb{R}^{n-1}}\frac{1}{\left(|z'|^2+1\right)^{\frac{n-2s_1+2}{2}}}\,{\rm d}z' \\ & \geq C\int_{2x_n}^{+\infty}\frac{y_n^{\left(\frac{q(2s_1+\alpha)+(2s_2+\beta)}{1-pq}+o(1)\right)p+\alpha+1}x_n}{(x_n+y_n)^{3-2s_1}}\,{\rm d}y_n. \end{align*}

Let $y_n=x_nz_n$, one has

(3.32)\begin{align} u(x)& \geq Cx_n^{\left(\frac{q(2s_1+\alpha)+(2s_2+\beta)}{1-pq}+o(1)\right)p+2s_1+\alpha}\int_{2}^{+\infty}\frac{z_n^{\frac{q(2s_1+\alpha) +(2s_2+\beta)}{1-pq}p+\alpha+1}}{(1+z_n)^{3-2s_1}}\,{\rm d}z_n\nonumber\\ & \cong\int_{2}^{+\infty}\frac{1}{(z_n)^{3-2s_1-\left(\frac{q(2s_1+\alpha)+(2s_2+\beta)}{1-pq}p+\alpha+1\right)}}\,{\rm d}z_n. \end{align}

Due to $p+2s_1+\alpha >1$ and $q+2s_2+\beta >1$, we have $\frac {q(2s_1+\alpha )+(2s_2+\beta )}{1-pq}>1$, which implies that

\[ 3-2s_1-\left(\frac{q(2s_1+\alpha)+(2s_2+\beta)}{1-pq}p+\alpha+1\right)<1. \]

This yields that the right-hand side of (3.32) is infinity, which is impossible. Hence, for $pq<1$ we also obtain $(u,v)\equiv (0,0).$

In sum, we conclude that there is no nontrivial classical solutions to system (1.1) for the case that $(p,q)\in \mathcal {R}_{sub}$, $\min \{p+2s_1, p+2s_1+\alpha \}>1$, $\min \{q+2s_2,q+2s_2+\beta \}>1$.

Proof. Assume that $(u,v) \not \equiv (0,0)$, from the proof of (i), we know that $u>0$ and $v>0$ in $\mathbb {R}_+^n$. Applying lemma 2.3, for $x_n>\min \{2,\frac {|x'|}{10}\}$, we have

(3.33)\begin{align} u(x)& \geq C\int_{\mathbb{R}_+^n}\left(\frac{1}{|x-y|^{n-2s_1}}-\frac{1}{|x^*-y|^{n-2s_1}}\right)|y|^{\alpha}v^p(y)\,{\rm d}y \nonumber\\ & \geq C\int_{B_1(2e_n)}\left(\frac{x_ny_n}{|x^*-y|^{n-2s_1+2}}\right)|y|^{\alpha}v^p(y)\,{\rm d}y \nonumber\\ & \geq C\frac{1+|x|}{(1+|x|)^{n-2s_1+2}}\int_{B_1(2e_n)}y_n|y|^{\alpha}v^p(y)\,{\rm d}y\nonumber\\ & \geq C\frac{1}{(1+|x|)^{n-2s_1+1}}. \end{align}

Similarly, for $x_n>\min \{2,\frac {|x'|}{10}\}$, we have

(3.34)\begin{equation} v(x)\geq \frac{C}{(1+|x|)^{n-2s_2+1}}. \end{equation}

Iterating (3.33) with (3.34), for $x_n>\min \{2,\frac {|x'|}{10}\}$, we get

(3.35)\begin{align} u(x)& \geq C\int_{B_{|x|}(0,4|x|)}\left(\frac{1}{|x-y|^{n-2s_1}}-\frac{1}{|x^*-y|^{n-2s_1}}\right)\frac{|y|^\alpha}{(1+|y|)^{p(n-2s_2+1)}}\,{\rm d}y\nonumber\\ & \geq C\int_{B_{3|x|}(0,4|x|)\setminus B_{2|x|}(0,4|x|)}\frac{x_ny_n}{|x^*-y|^{n-2s_1+2}}\frac{|y|^\alpha}{(1+|y|)^{p(n-2s_2+1)}}\,{\rm d}y\nonumber\\ & \geq C\frac{(1+|x|)^{2+\alpha}}{(1+|x|)^{n-2s_1+2+(n-2s_2+1)p}}\int_{B_{3|x|}(0,4|x|)\setminus B_{2|x|}(0,4|x|)}\,{\rm d}y\nonumber\\ & \geq C\frac{1}{(1+|x|)^{(n-2s_2+1)p-2s_1-\alpha}}. \end{align}

Using the same argument as that of (3.35), for $x_n>\min \left \{2,\frac {|x'|}{10}\right \}$, we obtain

\[ v(x)\geq C\frac{1}{(1+|x|)^{(n-2s_1+1)q-2s_2-\beta}}. \]

After $k$ iteration steps, it is easy to see that for $|x|$ large and $x_n>\min \left \{2,\frac {|x'|}{10}\right \}$,

\[ u(x)\geq \frac{C}{(1+|x|)^{\gamma_k}}, \quad v(x)\geq \frac{C}{(1+|x|)^{\delta_k}}. \]

Here,

\[ \gamma_k=\delta_{k-1}p-2s_1-\alpha,\quad \delta_k=\gamma_{k-1}q-2s_2-\beta, \]

where

\[ \gamma_1=(n-2s_2+1)p-2s_1-\alpha,\quad \delta_1=(n-2s_1+1)q-2s_2-\beta. \]

Simple calculations imply that

\begin{align*} \gamma_{2m}& =(n-2s_1+1)(pq)^m-\left[\left(p(2s_2+\beta)+(2s_1+\alpha)\right)\frac{1-(pq)^m}{1-pq}\right], \\ \gamma_{2m+1}& =\left[p(n-2s_2+1)-(2s_1+\alpha)\right](pq)^m\\& \quad -\left[\left(p(2s_2+\beta)+(2s_1+\alpha)\right)\frac{1-(pq)^m}{1-pq}\right],\\ \delta_{2m}& =(n-2s_2+1)(pq)^m-\left[\left(q(2s_1+\alpha)+(2s_2+\beta)\right)\frac{1-(pq)^m}{1-pq}\right], \\ \delta_{2m}& =\left[q(n-2s_1+1)-(2s_2+\beta)\right](pq)^m\\& \quad -\left[\left(q(2s_1+\alpha)+(2s_2+\beta)\right)\frac{1-(pq)^m}{1-pq}\right], \end{align*}

where $m=0,1,2,\ldots.$

From $0< pq<1$, we obtain

\[ \gamma_k\to-\displaystyle\frac{p(2s_2+\beta)+(2s_1+\alpha)}{1-pq},\quad \delta_k\to-\frac{q(2s_1+\alpha)+(2s_2+\beta)}{1-pq},\quad \text{as } k\to\infty. \]

This yields that for $|x|$ large,

\[ u(x)\geq C(1+|x|)^{\frac{p(2s_2+\beta)+(2s_1+\alpha)}{1-pq}-o(1)}, \quad v(x)\geq C(1+|x|)^{\frac{q(2s_1+\alpha)+(2s_2+\beta)}{1-pq}-o(1)}, \]

as $k\to +\infty$. Due to $0< pq<1$ and $\alpha \geq -2s_1pq$, $\beta \geq -2s_2pq$, we have

\[ \frac{p(2s_2+\beta)+(2s_1+\alpha)}{1-pq}>2s_1,\quad \frac{q(2s_1+\alpha)+(2s_2+\beta)}{1-pq}>2s_2 \]

This contradicts with the assumptions that $u(x)\in L_{2s_1}$ and $v(x)\in L_{2s_2}$. Hence, $(u,v)\equiv (0,0)$.